{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1b544a83",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T04:40:50.381050Z",
     "start_time": "2022-02-24T04:40:44.515643Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "860c228f",
   "metadata": {},
   "source": [
    "# Tensorflow操作"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89770a7b",
   "metadata": {},
   "source": [
    "## 张量和操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "491a81a6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:20:36.404914Z",
     "start_time": "2022-02-24T06:20:36.386722Z"
    }
   },
   "outputs": [],
   "source": [
    "t = tf.constant([[1, 2, 3], \n",
    "                 [4, 5, 6]])  # 2*3矩阵，类似numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7c81c79d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:20:36.729880Z",
     "start_time": "2022-02-24T06:20:36.710868Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=42>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(42) # 标量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "73799517",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:21:24.287336Z",
     "start_time": "2022-02-24T06:21:24.268742Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=int32, numpy=\n",
       "array([[2, 3],\n",
       "       [5, 6]])>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[:, 1:] # 索引类似numpy，先行后列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "de675752",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:30:35.228702Z",
     "start_time": "2022-02-24T06:30:35.212368Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=int32, numpy=array([2, 5])>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[..., 1] # \"...\" 表示前（后）全部维度，即替换若干冒号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7985f522",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:31:49.000261Z",
     "start_time": "2022-02-24T06:31:48.979292Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 1), dtype=int32, numpy=\n",
       "array([[2],\n",
       "       [5]])>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[..., 1, tf.newaxis] # tf.newaxis增加新维度，放在右侧为增加一个列维度，放在左边增加一个行维度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "323fcbe7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:43:58.548207Z",
     "start_time": "2022-02-24T06:43:58.539330Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[1, 2, 3],\n",
       "       [4, 5, 6]])>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "16c3b65e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:44:03.858086Z",
     "start_time": "2022-02-24T06:44:03.831359Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[11, 12, 13],\n",
       "       [14, 15, 16]])>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c2f4a4b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:51:26.030590Z",
     "start_time": "2022-02-24T06:51:26.018589Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[ 1,  4,  9],\n",
       "       [16, 25, 36]])>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(t) # 平方"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "69d88c0a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:51:47.381341Z",
     "start_time": "2022-02-24T06:51:47.359536Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=int32, numpy=\n",
       "array([[1, 4],\n",
       "       [2, 5],\n",
       "       [3, 6]])>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.transpose(t) # 转置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "87c2788f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:52:42.349219Z",
     "start_time": "2022-02-24T06:52:42.327435Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=int32, numpy=\n",
       "array([[14, 32],\n",
       "       [32, 77]])>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t @ tf.transpose(t) # 矩阵乘法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f2349119",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:57:38.267278Z",
     "start_time": "2022-02-24T06:57:38.253287Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=int32, numpy=\n",
       "array([[14, 32],\n",
       "       [32, 77]])>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.matmul(t, tf.transpose(t)) # 矩阵乘法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0cd7b468",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:56:22.925199Z",
     "start_time": "2022-02-24T06:56:22.899571Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=6>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.reduce_max(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2b51c070",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:57:12.885214Z",
     "start_time": "2022-02-24T06:57:12.873223Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=21>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.reduce_sum(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca47c558",
   "metadata": {},
   "source": [
    "## 张量与Numpy配合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "927848d2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:59:54.909240Z",
     "start_time": "2022-02-24T06:59:54.900797Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "fb56dbfa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T06:59:55.174090Z",
     "start_time": "2022-02-24T06:59:55.171600Z"
    }
   },
   "outputs": [],
   "source": [
    "a = np.array([2, 4, 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "67571224",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:00:13.910073Z",
     "start_time": "2022-02-24T07:00:13.901545Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=int32, numpy=array([2, 4, 5])>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "118d65f8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:00:28.374355Z",
     "start_time": "2022-02-24T07:00:28.355549Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 2, 3],\n",
       "       [4, 5, 6]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f987b4dd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:00:41.733933Z",
     "start_time": "2022-02-24T07:00:41.720789Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[ 1,  4,  9],\n",
       "       [16, 25, 36]])>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9b4c5b8e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:00:51.229997Z",
     "start_time": "2022-02-24T07:00:51.215696Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  4,  9],\n",
       "       [16, 25, 36]], dtype=int32)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa32b81b",
   "metadata": {},
   "source": [
    "## 类型转换"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38d80eae",
   "metadata": {},
   "source": [
    "必须要注意数据类型，比如浮点张量不能和整数张量相加，不能相加32位浮点张量和64位浮点张量。\n",
    "\n",
    "tensorflow默认使用32位精度，Numpy默认64位"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c930ad1d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:11:45.431373Z",
     "start_time": "2022-02-24T07:11:45.398971Z"
    }
   },
   "outputs": [
    {
     "ename": "InvalidArgumentError",
     "evalue": "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:AddV2]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_17380/2129067455.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2.\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m40\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# 报错\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow\\python\\util\\traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    151\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 153\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfiltered_tb\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    154\u001b[0m     \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    155\u001b[0m       \u001b[1;32mdel\u001b[0m \u001b[0mfiltered_tb\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mraise_from_not_ok_status\u001b[1;34m(e, name)\u001b[0m\n\u001b[0;32m   7184\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mraise_from_not_ok_status\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   7185\u001b[0m   \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m\" name: \"\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 7186\u001b[1;33m   \u001b[1;32mraise\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   7187\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   7188\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidArgumentError\u001b[0m: cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:AddV2]"
     ]
    }
   ],
   "source": [
    "tf.constant(2.) + tf.constant(40) # 报错"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "34681979",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:11:45.711397Z",
     "start_time": "2022-02-24T07:11:45.694385Z"
    }
   },
   "outputs": [],
   "source": [
    "t2 = tf.constant(40, dtype=tf.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b8f26ac3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:11:46.147579Z",
     "start_time": "2022-02-24T07:11:46.133426Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(2.0) + tf.cast(t2, tf.float32) # tf.cast() 转换类型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e3a35c7",
   "metadata": {},
   "source": [
    "## 变量\n",
    "\n",
    "tf.Tensor 自身值是不变的，tf.Variable则可以变化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "01e5a1a2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:17:45.197451Z",
     "start_time": "2022-02-24T07:17:45.163293Z"
    }
   },
   "outputs": [],
   "source": [
    "v = tf.Variable([[1., 2., 3.],[4., 5., 6.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "70d1bbb4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:17:45.896852Z",
     "start_time": "2022-02-24T07:17:45.872929Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b4b1a0a6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:19:30.543182Z",
     "start_time": "2022-02-24T07:19:30.529174Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 4.,  8., 12.],\n",
       "       [16., 20., 24.]], dtype=float32)>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.assign(2 * v) # 分配值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "da45528e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T07:19:53.548728Z",
     "start_time": "2022-02-24T07:19:53.526568Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 4., 42., 12.],\n",
       "       [16., 20., 24.]], dtype=float32)>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[0, 1].assign(42) # 指定位置重新分配值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "a066c67d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T08:23:23.695218Z",
     "start_time": "2022-02-24T08:23:23.676405Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[100.,  42.,  12.],\n",
       "       [ 16.,  20., 200.]], dtype=float32)>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.scatter_nd_update(indices=[[0, 0], [1, 2]], updates=[100., 200.]) # 指定位置和更新值"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "634cbc76",
   "metadata": {},
   "source": [
    "## 其他数据结构\n",
    "\n",
    "书337"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ae25fa2",
   "metadata": {},
   "source": [
    "# 定制模型和训练算法"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "296106db",
   "metadata": {},
   "source": [
    "## 自定义损失函数"
   ]
  },
  {
   "attachments": {
    "image-2.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAEDCAYAAAB0/A4MAAAgAElEQVR4nO3dd3xT5ffA8U9aCrRQllCGgCBQkI0oQ0AQQRARFQU3QxFE/X2ZDhQUlSEIyBAQJyDIUBCQvVr2UBAEUZC9N6UtHXTc3x+nhTKbtrm5yc15v155tU2T3HObJif3Puc5DyillFJKKaWUUkoppZRSSimllFJKKaWUUkoppZRSSimllFLKwzQCDKCgm7bXAYh207aUUin8rA5AKRuYCMy/yfX3IYm0lHvDUUp5Ok2+SvmG7FYHoJS6SpOvUu5zs1PKpVKuu++629YBtgFxwBag5nW/fwBYBcQAx4DxQJ40vw9PuW4YcAZYl4E4uwB7gcspX1+7ye/3pMR2BlgCZEv5XRVgBRAJRAHbgYcysG2lfIImX6U80zDgXSQp7wcWAEEpv6sCLAXmAdWA1kB14PvrHuMlwAE0ANo5ud2ngC+BkUBlYBQwDng85ff3AWOBj4HyQBNgcZr7/wScAGoBNYD+SJJWSimlXGoikIgULqW9xHDtmK8zR76pt3kxzW1yAxFAp5SfJwPfXRdD9ZT7haT8HA785UTs1xdcrePGJD4RWJvyfWvgIhB8i8eLBNo7sV2lfJoe+SrlGquRBJj28kIWHm9Dmu+jgR1AxZSfayJHtWkTfepp5TJp7rclE9u9hxtPUa9Ns+1lwCHgADAVSbRpE/EI4FtgJfABUCETMShle5p8lXKNGGR8NO3l6HW3SU756khzXUAmtuWHJLi0ib4aUA4ZJ051KROPDXIEfavrooB7gbbAYaAP8C9QLOX3/ZFEPQcZl/4LeCWTcShlW5p8lXKfMylfi6a5rvotblsnzfe5kPHXf1J+3gpU4sZkvxeIzWKM/wD1r7uuPrArzc+JyJFtH6BqSnwt0/z+P2A08BhyerwTSqlrZEv/JkopF9kLHEGODt9Dxnv73uK2fZFkfRz4EKk8/inld0OAjcBXwATkaLQCUhTVJYsxfg78jJyyXgo0R8afW6f8viVyans1cB6pZA5GknYgUij2M3AQKIwk7k1ZjEkppZS6QUaabDyAnBqORcZ1H+PmBVetkFO28ciR7v03eezFSIHTJWRM+JM0vw9HqpbTc7MOV68jHxQSuHGqUX0gDDiXsg87gY4pv8uOfEA4lBL3ceBrrp0CpZRSSimllFJKKaXUdcohE+anWB2IUkop5a0yWu08FvjdjECUUkopX5GR5Psc0mVnhUmxKKWUUj7B2eSbB6mk7GViLEoppZRPcHae76fIZPkj6dyuc8qFnDlz1ixZsmQWQvNsycnJ+Pnd+rOLYcC5czkoWDDejVG5Rnr75u3svH9HjhzBMAx8+bXn7bxx/+Li/MmRIxmH42bN0a7yxn3LiD179pwFCjlzW2eSb3Vk5ZIaTtz265QLJUuWNHbv3u1MDF4pPDycRo0apXu7mBgICkr3Zh7F2X3zVnbev0aNGhEREcG2bdvSv7GXsvPzB963fxs3QqVKEHyrpTbS8LZ9yyiHw3HI2ds68xGkEdIk4DBwEugNPI1M/Fe3ceECVK0KiYlWR6KUUuaYNAmOH7c6Cu/jzJHv18D0ND/3RpJxV1MispH8+WHnTsimTTyVUjY1frzVEXgnZ458Y5Aj3tRLNDLX98zt7qREQgK8/76MASullJ088QT8/bfVUXinzByT9Xd5FDaWOzeUKCGnngMys3icUkp5qC++ABvX9pnKvmVnHsLhgK5d4eRJqyNRSinXmTUL8uXTYbXM0uTrBrGx8Nhj8lUppexg0yYdTssK/cziBoGBsH27HAUrpZS3S0iAoUOtjsK76ZGvmyQlwbPPyrxfpZTyVoYBNWrAsWNWR+Ld9MjXTbJlg06ddHxEKeXdHA5Yvx7y5LE6Eu+mR75u1KQJbN6s4yRKKe81eLC+h7mCJl83++ILOKMzpJVSXigpCbJnh1y5rI7E++lJUDdyOKQ8XymlvNHZs9BL17ZzCT3ydTPDgKZN4fBhqyNRSinnxcdDw4YQHW11JPagR75u5nDA2LFQvLjVkSillPNy5IBdu8DGKwK6lf4ZLRAaCj//rNOOlFLe4fJlaN9e5vcq19Dka5Fdu+D0aaujUEop57RuLUe/yjX0tLNFPv5YFlswDO18pZTybDt2QKtWVkdhL3rka6HHHoMtW6yOQimlbu38eVkWNTnZ6kjsRY98LTRjhqwKopRSnqpAAViyxOoo7EePfC2ULx98+y3s3m11JEopdaMjR+DRR7WjlRlMS74REbpyvDPy59fSfaWUZypWDIYN07oUZ/zwQ8Zub9rb/unTOenWTdqRqVt7+mkoUgQiI62ORCmlrrp4EebNg0qVrI7EsyUnQ58+8MorGbufqcdco0fDE09AVJSZW/F+ffvC4sVWR6GUUledOiVTItWtxcTIUrGffQb+/hm7r2nJt0SJGAoUgAULoEEDGTtQNzdyJLRta3UUSiklkpKgTBn44AOrI/FcJ0/CQw/BL7/I8ooLF2bs/qYl38DAJDZtkm5O27dD7do6reZWHA6YPBmmTrU6EqWUgqVLoV07q6PwXDt3Sk7bvBlKlZL1jR95JGOPYepp57JlYcMGaNQITpyQI+A5c8zcove6/3544AGro1BKKalwHj/e6ig80+LF8l59+DDUqQMbN2ZuXNz0OtvUOWIdOkBsrLQoGzZMS9evd889VxuXK6WUVTZtkh4EefJYHYnnGTdOmiNFRclQ4cqVULhw5h7LLZNcsmeH77+HQYMk6b79NnTpok26r7duHYSHWx2FUsqXBQXJFEh1VVIS9OgBb74p1c19+8K0aRAYmPnHdFuHK4dDyrHLlpWxhG++gf37ZbBauzyJNm3kq/Z7VkpZ4exZKbSqUsXqSDxHdDQ8/zzMnw8BAZK72rfP+uO6vb1DmzZydBcSAitWQN26koSVWLgQuna1OgqllC/6+Wf44guro/AcR49KrdL8+XI2YNky1yResKi3c+3aMq7QsiX8/bf8PHeuFhyBPNF16lgdhVLKF3XtqvU4qbZuhccfh+PHoVw5ScChoa57fMsaG5YqJWOczZrJqY7GjeUcuq8LDpbTHLNmWR2JUsqXjB4tB0E65CV/hwYNJPE++KDM2nFl4gWLF1bIm1c+TXTtCvHx8MIL8Mkn+skrKUlOdyillLs0bw7Vq1sdhbUMA0aMgKeeku5V7drJnOc77nD9tixv6Z8tG4wdK+MMDgd89JHscHy81ZFZp3Rp6NZNeqsqpZTZNm+WMc277rI6EuskJMiBYK9ekoQHDICJE2UKqBksT74gSbd7dznUz5ULpkyBJk3kdLSvOnBATsX7+lkApZT5liyB//6zOgrrXLwoNUgTJkiynT5dWmuaeQreI5JvqscfhzVr4M47Ye1aKTzy1bVuS5eWcQYdf1FKmSk5Gfr1892C14MHZd+XLoVChSAsTBZLMJtHJV+AGjWkErpGDdi3TxJwWJjVUVnj8mV47TVdllEpZZ6HH5ZZJ75o40aZbbNrF1SsKLmnbl33bNvjki/Ike+aNdCqFURESMPqjC5UbAe5ckkRRHKy1ZEopexqxgxJPL5m5kxZlej0aWjaVGbflC7tvu17ZPIFSTyzZ8vgd2KiLFTcp49vJSKHQ3phb9miY79KKdcbOlTGOH1peMswpNXxs89CXBx07ixL37q706LHJl+QxYmHDYOvvpLvP/tM/mAxMVZH5l5Dh8rakUop5SqJiZKIcuWyOhL3uXwZOna8Wkw1fLjkl4AA98fi0ck3VZcusGiRrLLxyy9yqsBXkpHDIWcAihSxOhKllJ2cPg3vvivTPX3B+fMyhDlpkiweMXs29Oxp3VG/VyRfkHPyGzZIZ6zNm2WQfMcOq6Nyn7Zt4Y8/rI5CKWUH58/Lmr2JiVZH4h7//SfFu6tWQdGisHo1PPmktTE5m3ynACeASGAP0Mm0iG6jYkWpTqtTRxYyrldPFjb2BZ9/Dvfea3UUSik7KFAAtm3zjaPe1aslZ/z3H1SrJgdvNWtaHZXzyXcwUArIA7QCBgCWhF+4sCxg/OyzsqDxY4/JAsd2V6qUrHjkq/OelVKusX8/dOrkG0VWkydLw6bz56WJxtq1ULy41VEJZ5Pv30Bqw0cj5VLGlIicEBgIP/0kCxonJ8sCx927238+7MWL8oFDKaUyq0gRePVVq6MwV2rjkPbtpW1kt24wZw7kzm11ZFdl5LPPOKADEAj8CTwIRF93m84pFwoVKlRz5syZrojxtpYsKcywYeVJTPSjbt2z9O37D0FB5mfh6OhoclvwTCYlOYiP9zN1H63aN3ex8/51796dpKQkxowZY3UoprHz8wfm7t+FCwGcOBFIxYqRpjx+etzx3F2+7Mdnn1UgLCwEPz+Dt976j6eeOm7qNlM99NBDW4D7zHhsf6A+0Be4bXF2aGio4S6rVhlGgQKGAYZRrZphHDli/jbDwsLM38hNDBhgGMOHm7sNq/bNXey8fw0bNjSqVatmdRimsvPzZxjm7t+GDfIeYhWzn7tTpwyjbl3JBcHBhrFokambuwHgdFlsRqudk4C1QHGgawbva5oHH5RCrHLlYPt2qFVLGlPY0TvvSHm8UkplRGKiFB598IHVkZhj1y6ZBbNhA5QsKR2rmje3Oqpby+xUo2xYOOZ7M+XKSQJu2BBOnJCEPGeO1VG5XkCA/FMNGWJ1JEopbzJqlDQqsqNly6Qn88GDcP/90qO5ShWro7o9Z5JvCPAckBs57dwMeB5YaWJcmVKggKxM0b69dMFq3Vo6mNitNePdd0OjRlZHoZTyJt26yXq1dvPNNzJnOTISnn4awsO9oymRM8nXQE4xHwUuAMOA7sBcE+PKtOzZZRGGgQMl6fbuDa+/LhVvdlG0qHyqW7PG6kiUUt5g0iT46y/Im9fqSFwnKQneflt6MyclwXvvyWIJQUFWR+YcZ5LvGaAhkA+Z51sF+MbMoLLK4YD335fVOnLmhK+/hhYtZIUkuzh3Dr7/3uoolFLeoEABeyXeS5fgmWek93+2bPDttzB4MPh5Tc9GL2ovmRlt28pawCEhsHy5LJh84IDVUblGiRJyhG/3uc1KqazZvVuaEZXxqCqdzDt+XGp75syRlYiWLPHOecu2Tr4g1X2bNkGlSvDPP1er4ewgNhYqV/a9VZ6UUs7r1Qv27bM6CtfYvl3ew7dskdqXDRugcWOro8oc2ydfkNaM69bJihZnzsiqSNOnWx1V1gUGyrivt4xxKKXcb/58mQ3i7RYskH7+R4/K102boEIFq6PKPJ9IviDjHQsWSPFVfDw8/zwMGOD9ldAFC8p+nD9vdSRKKU8SEwP169vjzNjo0dCqlYz1vvCCDCMWLGh1VFnjM8kXZGB+3Dj44gspykrt/Rkfn/59PVnJkr6zNJhSyjlBQVJs6s1nxhIT4a23ZJpUcjL07w9TpkghrbfzqeQLknS7d5fB+ly54McfZa3gc+esjizz2rWTf8yLF62ORCnlCS5dgu++k2VYvVVkpBztjh0rU0inTIGPPrLPakw+l3xTtWol46XFisnXOnW8e7m+QYNg/Xqro1BKeYLz5+HsWaujyLzDh+WU+aJFcnp5xQp48UWro3Itn02+ADVqyMLKNWrA3r3Sniw83OqoMmf0aOnyopTybXFxcMcd8O67VkeSOb//Lv35d+yA8uWlbXD9+lZH5Xo+nXwB7rwTVq+WI+ELF6Qi+ocfrI4qcyZPhqFDrY5CKWWlpUtlnNQbzZolc3hPnZJZKRs22Gd+8vV8PvmCLLA8e7asFpSQAK+8Ih2ykpOtjixjmjSR2JVSvqtVK5gwweooMsYw5MDhmWekf8Grr8LixZA/v9WRmUeTbwp/f1mEYfx4+X7wYHj2WflH8BbFiknl9s8/Wx2JUsoKo0fDvHmy+pm3uHwZXnvt6mnyIUNksYTs2a2Ny2zZrA7A07z+unROadMGfvlFBv7nzYPCha2OzDnJybKsllLK9zRr5l3TcC5ckKPdlSsl7ilTZGUiX6BHvjfxyCNSOXzXXVKQVbs27NxpdVTOKVFCVvo4dcrqSJRS7rRkiRwk3HWX1ZE4Z98+KXJduVLiXrXKdxIvaPK9pUqVpH1Z7dpw6JAsyrBkidVROSc6Gh5+WKoelVK+YelSmRvrDdatuzq9s0oVOcipVcvqqNxLk+9tFC4sqyK1bQtRUbIyyPjxVkeVvty5Ze1Obzr9pJTKvEuXpGalZEmrI0nfTz/JYghnz0Lz5rB2rXfE7WqafNMRGAjTpsEHH8jyfW+8AT16eP5Sfn5+0KGDdzcOUUqlLyoKqlf3/DNdhgGTJt3Fiy9KkdWbb8Jvv0GePFZHZg1Nvk7w85PFCyZOlCrCkSPhww8rEx1tdWS399ZbsqKTUsq+goNlqT1PPtMVHw8vvwwTJ5bGzw9GjYIvv5R++75Kk28GtG8Py5bJ3LP16wvSoIEsb+Wp7rtP1r385x+rI1FKmWHPHnjvPc9ePOHsWelBMHUq5MyZxNy58L//WR2V9TT5ZlDDhtLurHjxGLZtk4KsrVutjurW9u+H48etjkIpZYaCBWV2hqf69195j1y7FooXhzFj/qRlS6uj8gyafDMhNBS+/HIrDz4oia1BA5g71+qobu6ll6S4QVc8Uspe9u6V1dgaN7Y6kptbuVKmEu3fD/feK7NHypb18LE6N9Lkm0l58yaydKmcio6JgaeeghEjpKjA08yZA716WR2FUsqVtm3z3IVgvv9eGn5ERMATT0j//GLFrI7Ks/jwcHfW5cghizCUKwd9+0qC27MHxozxrPZurVrJRSllD5cvS2coT5OcLH3xhwyRn3v3hs8+k5a96lp65JtFDodMQ5oxQ5LxhAkyH9iTTvP6+0vRwwsveN9iEUqpGz3/vOcd9cbESE+EIUPkPWfCBPj8c028t6LJ10XatpUXQ6FCUhH9wANw4IDVUV0VEgKdOsmHBaWUd5s0SWpNPMXJk9CokSwJmCcPLFoEnTtbHZVn0+TrQnXqSFFBxYqwa5dU+W3YYHVUwuGQwoyZM71rpSal1FWJidCli3zvKUeUO3bIe93vv0tfgQ0boGlTq6PyfJp8Xax0aVmUoWlTOHNGFoSeMcPqqK7auVMXXVDKmzVtCrlyWR2FWLwY6tWT1d/SHnyo9GnyNUHevLBggXxCjY+H556TDlmeUAn96adSdRgVZXUkSqmMuHRJPtg/84xnDB+NGyf1LVFRsvb5ypUyvKWco8nXJAEBsgjD8OHyQunXT3otx8dbHZkk4ClTrI5CKZUR+/dLL2SrJSVB9+7Smzk5WWZ6/PST9MFXztOpRiZyOKBnTyhTRiqNJ0+Whe5nz4Y77rAuro8+8u2eqkp5m7g4qFxZqoetFB0tldbz58sBxrffQrt21sbkrfTI1w2eeALWrJHTvatXy9jInj3WxZMtm7TEfPNN62JQSjlv2DBZ0MVKR49KhfX8+VCgACxfrok3KzT5uklqe7Xq1aUtXN26sGqVdfFUqCBTj5RSnq9Pn6tVzlbYskUWu9+2TZoKbdwIDz5oXTx2oMnXjYoXlyPgxx+H8+elanHSJGtiCQqCSpXktJEnFIIppW7uf/+DQ4esW7lo7lxJtCdOyNcNGyQBq6zR5OtmuXPDr79Cjx6QkCBFWH37WtN5yt9fTn/HxLh/20op57RoIR/c3c0wpGD0qafkPSJ1SVUr61XsRJOvBfz9ZRGGcePk+4EDpYjB3c0v/P1h6FDZ7uXL7t22Uur2EhOlKU6zZpA9u3u3nZAAXbtKb2bDkKmSP/zg/jjsTJOvhbp2lfnAwcHyInvoIWsaYHTvDuvWuX+7SqlbO3NGxlbd7eJFmb87YYL0q58xQ/rXe8LcYjvR5GuxZs1k4vxdd0lBVu3a8Pff7o1h8mRJ/EopzxAdDfnzyxkydya9AwekL/2yZdKnPjxc+tYr19Pk6wEqV76aeA8dkn/+JUvct30/PzkC79nTfdtUSt3azJkyH9+dNmyQ96Bdu6RF5KZNMi1SmcOZ5JsD+A44BEQBfwKPmhmULypcGMLCoE0biIyU0z5ffeW+7devL+M7SinrvfKKjLO6y4wZcvbrzBmZhbF+vfSpV+ZxJvlmA44ADYG8QD9gJlDKxLh8UmAgTJ8ui1EnJcmYcM+e8r3Z8uaVsedBg3TqkVJWGjq0PNu2SQcpsxmGFHw+95y0vu3SRc6C5c1r/rZ9nTPJ9xLQHzgIJAPzgQNATRPj8ll+fvJi+OEHefF98QW0bi1jQGYLCpICi8REraxQyipPP33ULSsDxcdfnerocMi0ovHj3ZP0VebGfAsDoYCby4J8S4cOsHSpFF3Mmydt3Y4eNXeb/v7QqxecOBGoc3+VcrP4eEmApUpdMn1Kz7lz8MgjUmwZFCS9B3r21Ipmd8poe/0AYCowCfj3Jr/vnHLhwoULhIeHZy06DxYdHe2W/Rs1KpA+faqwbVsQNWrEM2jQDsqVM/cweNq00kRHb6FiRXuuO+iu584KERERJCUl2Xb/wL7PX2RkNvbtK0L58ubu39Gj8p5y9GgQBQvGM3DgDvLmjcYdf1K7Pndm8wOmAwuRJHxboaGhhp2FhYW5bVtnzhhGgwaGAYYRFGQYc+eau73UfYuPN3c7VnHnc+duDRs2NKpVq2Z1GKay4/N38qRhnDol35u5f+HhhlGggLyXVKtmGEeOmLapm7Ljc5cW8EdGEqozHEjFc2HgaSDB2Q2orCtYUObdvfyytHl78kkZCzazMGrOHHjjDfMeXyl11YoV8M035m5j8mSpZD5/Hlq2hLVrrWlbqYSzp53HA/cATQA3N0FUIIVQkyZBaCj06yfjM3v2wJgx5qzN26KFjAkppcyVmCjrfZslOVnmDKdOXereXZYo9Pc3b5sqfc4c+d4FdAGqAyeB6JTLiybGpW7C4ZDKxGnTJBl/9ZXMB7540fXbSi34ePFF9/ecVsqXNGsGO3aY89ixsZLYBwyQmRRjx8pZM0281nPmmOkQctpZeYjnnpN2lE88IRXR9erJAtelXDzzOihITnVrM3WlzDN9ugwtudrp0/IesXHj1f7xzZu7fjsqc7S9pJeqW1fav91zj/SCrl3bnCbszZtLf9f9+13/2Er5siNH4M03JfG6eorPrl1X3xNKlpSFUzTxehZNvl6sdGlpA9e0qXzKfegh+XTragcPWrPaklJ2VqCArJXr6sS7bJl8OD94EO6/Xz6kV6ni2m2orNPk6+Xy5ZN2cJ07Q1wcPPus61tEvvqqfIo+cMB1j6mUL1u7Fg4fhiZNXPu4X38Njz4q/eGfeUbOWhUp4tptKNfQ5GsDAQFSfDV8uHyK/uAD6NgRLl923Ta2bdOFF5RylcOH4cQJ1z1eUpK8Prt0ke/fe08WSwgKct02lGuZMElFWcHhkOlHd98tFcqTJslpp1mz4I47sv74994Lv/wiL2ytlFQq8w4ccO3UokuX5DU/d65MO5wwQVZFUp5Nj3xt5sknYc0aKFYMVq2SsZ///nPd4zduLPOLlVIZd/myLE4fEeGaxzt+HB58UBJvvnwy+0ETr3fQ5GtD994rRRbVqknirVNHEnFWORxS0BUamvXHUsrXJCfLkenmzZIos2rbNqhVC7ZuhTJlYMMGKbpU3kGTr00VLy5FHS1bSju5pk2lvVxWFS4MixfLZH2llPNmzpRVw1xR3Tx/PtSvD8eOyTz/jRuhQoWsP65yH02+NpY7t/Ro7t4dEhKgfXvpkJWcnLXHrVBBXvhKKee1aSPFkFlhGDBqlDTPSB3rXbHCnCYdylyafG3O31/ayY0dK98PHCjFHllpGVmqlDT3+PZbcxd3UMoODEOaaRw4kLUkmZgI//d/8mE6ORk+/hh+/FFazSrvo8nXR7zxhpyqCg6WKQiNG0tjjszy84O9e2WVJaXUrTkc0kyjZMnMP0ZkJDz+uHyIzp4dpk6FDz90fYMO5T6afH1I8+bSZq5kSRkjql1bWlNmRrZs8NlnEB0NJ0+6Nk6l7OLcOZn216RJ5nukHzok47qLF8uR88qV5q6CpNxDk6+PqVJFKqFr1ZJ5wA88IO3oMmvSpKzdXyk7i4iQo9bM+v13+ZC8c6fUWmzcKIlYeT9Nvj6oSBEIC5P2c5GR0o5uwoTMPdY778jKR5cuuTZGpbzdjh0y3/7//i9z9581Cxo2lL7qjRtLH/cyZVwbo7KOJl8fFRQkY799+kjXqtdfl2kQSUkZf6zTp2UucWKi6+NUylt98w38+WfG72cYMGSIfDiOjZXe6osXQ/78ro9RWUfbS/owPz9ZhKFcOVmYYcQI2LcPunTJ2GeykBA5HZZN/5uUAuR08+jRGb/f5cswbFh5Fi6Un4cMgbff1sIqO9IjX0XHjtKWLn9+aVPXrVsNjh3L2GPkyiXVl7NnmxOjUt5i925pbpPRaXgXLkhR5MKFRQkMlNPO77yjideuNPkqQNrSbdggY0r//RdM7doZP2XWvj00a2ZOfEp5A8OA8uWl8UVGkua+fdKHPSwMChSIZ9UqaN3avDiV9TT5qivKl5fTx1WrRnDsGDRoAL/95vz9y5SRqUfvvKPNN5RveuMNWLIkY40v1q6Viubdu2U2wrhxW7n/fvNiVJ5Bk6+6RsGC8Pnn23npJalgfuIJGDnS+WRaoABUr25ujEp5qn79ZJUhZ02dCg8/LPOBH31UEnHhwvHmBag8hiZfdYPs2Q0mT4ZPPpGk26OHtMdzppo5IEAaACxfLh2wlPIFR49C165QtCgEBqZ/e8OA/v3hpZekyOqtt2DePMiTx/xYlWfQ5KtuyuGQT/E//SSn0MaPlyKSixedu//Ro3DmjLkxKuUp7rhDWkg6M84bFydJ9+OPZcbB6NEwZozOFvA1mnzVbT3/vLSzK1RIxrLq1ZPOWOnp2FHGsX7/3fwYlbLSTz/BkSPwyCPp3/bMGWk1+dNPsurYvHmZb8KhvJtln7UiIyM5ffo0CQkJVoWQJXnz5uWff/6xOgxTXL9vISEBrFkTwhICGTMAABtESURBVFNP5eHvvyWpzpsnX2/n9GkYMECmH/n7mxy0UhaJi5PhlvT8+y889hjs3y/rbc+fD9WqmR+f8kyWJN/IyEhOnTrFnXfeSWBgIA4vnMgWFRVFcHCw1WGYIu2+GYZBbGwsx44dY/lyaN8+D8uXQ6NGMHmyrFF6K0WKyLzh6GgZ47Lpn0v5qLg4WajklVfSv+3KlfD009J8o2ZNmUVQtKj5MSrPZclp59OnT3PnnXcSFBTklYnXlzgcDoKCgrjzzjuJiTnNwoXw2mvyxtO2LQwenH4l9JAhMHOme+JVyl0OHZKj1/R8953Mf4+IgCefhFWrNPEqi5JvQkICgc6UBCqPERgYSEJCAgEBsgjDsGFSXPL++/LJ//LlW9+3f3/pT3u72yjlTfbskbasX3xx69skJ8N770GnTjJT4O23pWtVrlzui1N5LssKrvSI17ukfb4cDlmEYfZsWaBh4kQpNjl//ub39feXU881aujqR8oePvhAlvm7lZgYOTM0ZIj8/3/9NQwdKtXNSoFWO6ssePJJWL1aTqGtWiXt8f777+a3zZ1bGgjkyqXdr5T3SkyUZThnzoSqVW9+m5MnpSZi1izIm1dWJHrtNffGqTyfJl+VJTVrwubNUrW5Z48sLbhmzc1vmz+/TLH45BP3xqiUqyxcCD173no+744dV6fYlS4ta/A2aeLeGJV30OSrsqx4cUm4jz0mp54ffhh+/PHmt23eXNYOVsrbJCZCq1YwbtzNf79okcyDP3xYzgJt3AgVK7o3RuU9NPlmUKNGjXjrrbcsf4z0XLhwgcKFC7Nv3750b/vMM88wYsSILG0vODh1OUJISIB27WSJwetPMRcoIOv/duwo1aJKeYPERLj/fjh7FrJnv/H3Y8dKB7ioKHjuOZlaFBLi/jiV99Dka1ODBg2iRYsWlClTJt3bfvTRRwwYMICLzvaOvAV/f1mE4csvpbDk00+lz3Nc3LW3czgk+RYrlqXNKeUWhiGtH5cskYVH0kpKgu7dpTdzcrK0ZJ06FXLmtCZW5T00+drI5ZS5PDExMXz77be8+uqrTt2vSpUq3H333UyZMsUlcbz5psx/DA6G6dOhcWPpdpXWgw/Kke+gQS7ZpFKmGThQKvqvP5KNipJVv0aNkg5XqYuRaEWzcob+m2RCcnIyH3/8MQULFiQkJITevXuTnJwM3PyUcocOHWjZsuU11yUmJtKtWzfy589P/vz5efvtt688BkhnqaFDh1KmTBkCAwOpUqXKDcmxUaNGdO3ald69e1OoUCHq1asHwMKFC/Hz87vyM8DQoUNxOBw3XD788EMAWrVqxbRp01z2N3r0Uen+U7IkbNggRSi7dl17m0KFZK6kUp7s9ddlrDeto0dlvesFC2QoZflyePlla+JT3skjkq/DYc0ls6ZOnYq/vz/r16/nyy+/ZOTIkcyYMSPDj5GcnMyGDRuYMGECX3/9NSNHjrzy+759+/Ldd98xduxYdu3aRZ8+fejSpQsLFiy45nGmTJmCYRisWbOGyZMnA7BmzRpq1qx5zdzcrl27cuLEiSuXXr16UaRIEdq1awdArVq12Lx5M7GxsZn9s9ygShXYtEnGyg4elCKUZcuu/j5vXmlPOXs2bNvmss0q5RL798OLL8qKRQUKXL1+yxaoVQu2b4fQUCmsysgavkqBhQsreLOKFSvSt29fgoODCQ0N5ZtvvmHFihU8//zzTj9G0aJFGT16NA6HgwoVKrBnzx5GjBhBz549uXTpEiNGjGDp0qU0aNAAgNKlS7N582bGjh3LY489duVxSpcuzfDhw6957EOHDlH0uv51wcHBV/o1DxkyhGnTphEeHk7ZsmUBKFasGAkJCRw/fpwQF1aKFCkC4eFSgDVrlhwRjxsHnTtfvY2/v879VZ6nZEkpIEz7QX3OHEnIMTFX5/KmTcxKOcsjjnwNw5pLZlW9bnZ9sWLFOH39oGY66tSpc82Rad26dTl27BiRkZHs2rWLuLg4mjdvTu7cua9cxo8ff0P1cs2aNW947NjYWHLeouJj8ODBjB49mrCwMMqXL3/l+tR2n6488k0VFCRNCd57TwpUunSB3r3le5Bxs6pV4ZtvpKpUKSsZhszlPXRIjnBTrxs+HFq3lsTbvr0UYGniVZnl7JHvW0AHoAowLeV7nxVw3fphDofjynitn58fxnWZPaPLJqY+1m+//UbJkiVvu+1cN2kUW7BgQS5cuHDD9QMHDuSrr75i1apVV454U51P6Q1ZqFChDMXqLD8/WYShXDlJvsOHw759MGXK1a5XBw9K+8m8eU0JQSmnOBzQtCnceaf8nJAg1cxffy0/DxokHyS1Q67KCmePfI8DA4DvTYzFFgoVKsSJEyeuuW779u033G7Tpk3XJOmNGzdSrFgx8uTJQ8WKFcmRIweHDh2ibNmy11zuuuuudGOoUaMGu66rbvr000+ZMGHCNaea09q5cyfFihWjcOHCzu5qprzyCixdCvnyySm8Bx+E48dlKsfAgXLku2KFqSEodUtz5kgNwqOPynShiAho0UISb86ccganTx9NvCrrnE2+s4E5wDkTY7GFxo0bs2jRIubNm8fu3bvp2bMnR44cueF2x48fp3v37uzevZtffvmFzz//nB49egAyPtu7d2969+7N999/z969e9m2bRtfffUVX6d+/L6NZs2a8c8//3DunDxdAwcOZNSoUUyfPp1cuXJx8uRJTp48SVyaCbhr1qyhefPmLvor3N5DD0mRSpkysHWrnNpLLbg6dkx6QCtlhbvvhlKl5PsDB6Rj1fLlMs0oPPz261crlRGuLrjqnHLhwoULhIeH3/RGefPmJSoqysWbdo+kpCQuX75MUlLSlX1ISEggMTGRqKgo2rRpwx9//EHHjh0B6NSpEy1btuTcuXNXbp+UlETbtm2JjY2ldu3aOBwOXn75ZTp16nTlNu+88w558+Zl6NChdO3aleDgYKpWrUq3bt2ueZzLly/f8LcsVaoUNWvWZOLEibz22msMHTqUyMjIa6YeAcybN49GjRoRFxfHr7/+yuzZs4mKirpm39KKi4u75XOaGcOHB9CvXyV27MhH3bpJfPjhLurWPUfDhvDtt7nJkyeBkJB4l20vVXR0tEv3w5NERESQlJRk2/0Dc56/iIgA5s4tRrt2h3A4YOzYPPTtW5mIiOyUKnWJwYN3EBsbhzv+rHb+/7TzvpltADDRmRuGhoYat7Jr165b/s5bREZGWh3CbS1atMgIDQ01EhMT073tl19+aTRt2vTKz7faNzOet7g4w3jpJSmB8/MzjJEjDSM52TBGjzaMRYtcvjnDMAwjLCzMnAf2AA0bNjSqVatmdRimMuP5O3/eMCZNku+nTzeMHDnkf/KRRwwjIsLlm7stO/9/2nnfDMMwgD+cTaYeUe2sXK958+a8+eabHD16NN3bBgQEMGbMGDdEdaMcOaQz0McfS3u+1FZ9XbvKIgwrVlytilbK1QxDCgENQ5pkDBggvZnj46W5xoIFWgCozKHzfG3sf//7n1O365x20q0FHA5ZhKFcOejQQeYB798P06ZJNXSFClcrT5VyJcOQNqgOh/zvTZ4s348YceMcX6Vcydnkmy3l4p9yyQkkplyUconnn5fGBk8+KQuQN2ggPaILF5Zil0aNrI5Q2cnUqXDPPfJ/9+STsHq1zEmfNu3GdpJKuZqzp537ArHAe8BLKd/3NSso5bvq1ZOWlBUqwM6d0hN60SJpbK9dsJQr5c4tC37UqSOJt1gxWZdaE69yB2eTb3/Acd2lv1lBKd92992yGMPDD8OpU9C2rayVeuYM/P671dEpb/fXX3J6OV8+aRW5dy/UqAGbN8O991odnfIVWnClPFK+fHLE26mTrAfcpg188AGEhVkdmfJ2OXPK4ghNm8L58/D443Lkq3UFyp00+SqPFRAgnYWGDpXCl2+/hX//lSYcThRxK3WNkyfh/fdh0iQYPVraRvboAb/+KqeglXInTb7Kozkc8PbbsnpMYCD88AN07Ah/OD2bTimRLZucORk0SFbSGjdOqpr9/a2OTPkiTb7KKzz1lJwaLFJExujefVfeOC9etDoy5eni42VJy+bNpa1pcLDM3+3a1erIlC/T5Ku8xn33SVFM1aqwZw/07QvLllkdlfJ0//0nU9e2bIG77oL166FZM6ujUr5Ok6/yKiVKyJhvixYQGwsvvCDV0CYsQ6xs4NFHZSrRmTMybW3TJqhc2eqolNLkm2GtWrUif/78vPzyy1aH4rOCg2HuXPjf/6Ro5uefpS2gzgNWqQwDJkyQ5SsvXZJq+bAwadiilCfQ5JtBPXr0YPLkyRm+35EjR2jUqBEVK1akWrVqzJ4924TofEe2bDBqFIwZA35+UkRToYK80SrflpQk83Vff136hb//PkyfLgV7SnkKTb4Z9NBDDxEcHJzh+2XLlo2RI0eya9culi1bRrdu3YiJiTEhQt/y1lvw22+QK5eMAz/yiJxiVL7p4kVo3VrWh86WTarjBw6UD2hKeRL9l3STokWLUr16dQBCQkLInz8/Z8+etTgqe2jRQopoSpSQr6VLwz//WB2Vcrdjx6RX87x5kD+/FON16GB1VErdnCZfC/zxxx8kJCRQokQJq0OxjapVpZjmvvvk1HPdurBkidVRKXfZsEGe+xMnoEwZmVKkC3EoT6bJ183OnTtHu3bt+O6773DoemUuVbQorFoFTz8tpx8ffRS++cbqqJTZfvtNEu3Jk7IS1saNEBpqdVRK3Z4mXxcaOnQoDofjhsuHH34IQHx8PE899RR9+vThgQcesDhaewoKgpkz4Z13pOK1c2d44w0pvFH2YhhS5d6qFVy+DC+9JKeaCxa0OjKl0qfJN4OaNGlCmzZtWLp0KcWLF2fDhg1Xfte1a1dOnDhx5dKrVy+KFClCu3btMAyDDh060LhxY52mZDI/PxgyRHpB+/nB+PFyNKyV0PaRmCjFdv36yc+ffiorFeXIYW1cSjkrm9UBeJvly5cDEBUVdUPVc3Bw8JXrhgwZwrRp0wgPD6ds2bKsXbuWGTNmULVqVebMmQPAjz/+SJUqVdy7Az7k1Vel+Kp1a5gzR8YEV6ywOiqVVZcu+dOoEaxbJ8n2hx/g+eetjkqpjNHka4LBgwfz5ZdfEhYWRmjK4FP9+vVJ1nOfbte4sYwBNm4sKyLVrg0ffZRLi3G81KFD8NZbNTh4EPLkkWUndQRHeSOPOe3cv79cQIol9uyRXqw1a8p1vXrB8OHyfbFicPw4hIdfrWjs3FmWnwPpgBQVJYUYjz8u173wAvz0k3yf2TqntOO4efLkuWFsF2DgwIGMGzeOVatWXUm8yloVKsgC6vXqyVKEb7xxLwsWWB2VyqjNm6F6dTh4MDf33AN//qmJV6kbhIaGGreya9euW/7Okx0+fNho2LChcc899xiVK1c2Zs2adc3vP/nkE6NEiRLG3r17LYrQNSIjI296vbc+b6liYw3j1UcOG8U5bDgchjFihNURuV7Dhg2NatWqWR2Gy02caBg5chhGcQ4bD4fuNC5csDoi84SFhVkdgmnsvG+GYRiA04udesyRrzdI26Vq7ty513SpGjhwIKNGjWL69OnkypWLkydPcvLkSeLi4iyOWqXKmRO+WVyCJh0SMQzo2VOKdhITrY5M3UpiojxPHTrI0oAtOpfgvbFnyZfP6siUyhpNvhmQtktVoUKFrnSpMgyDoUOHcu7cOerVq0fRokWvXNatW2dx1Cotx8wZ9C7xA5MnQ0AAjB0rq96cOGF1ZOp6Z89C/frwxRfSKnLsWJjQeAZFV2vVnPJ+WnCVSVu3br3SpcrhcHBRV3X3DuPHc2dEBJW2fUKZMjJHdMsWqFEDZsyAhg2tDlABbN0q08NSC6sWLJBETCN5/vjkE6tDVCpL9Mg3E86dO0eXLl20S5WXe+AB+PtveOghOHVKvg4dqksTWskwZLWq+++XxHv//fIc1a9vdWRKuZYm3wxK7VLVs2dP7VJlA4ULy5qvPXrIG/+770pbytOnrY7M95w7B02aQPfu0pHs1Vdh9WooXtzqyJRyPU2+GWCk6VL1vM7qt41s2WDECJg7V05xLlkClSuj05HcaM0amUa0cqUsDzlrlnQoy5nT6siUMocm3wxYt24dM2bMYM6cOdSrV4/q1auzY8cOq8NSLtKqFezYIc35z5yBli3l6EuXXTZPbKzM4X/wQZmDXbcu7NwpXcmUsjMtuMqAtF2qbtZeUnmBX37h73XrqHeLX5csKc1bPv8c3n8fvv9eTn3+8IOOO7ra+vXw4osytutwSBIeNEiq0G8pnedPKW+hR77KtxQsSELevLe9iZ+fjP1u3QqVKsHevXI0/PrrEBHhpjhtLCYGeveWjmMHD0L58rIW8+efp5N4wannTylvoMlX+ZaJEymyeLFTN61WDf74A/r0kXHhCRPg7rtlPFIrojPOMGRcvVw5aRXrcMiHnG3bpKrZKRl4/pTyZJp8lW/J4Jt3zpxyKnTbNqhaFS5cgGeekZ7i//xjYpw2s3evzKF+8knpy162rCx48dlnGSyq0uSrbEKTr1JOqFRJGvmPHy/VuKtXQ5Uq0p7y7Fmro/NcFy/CO+9AxYpS0RwYCKNHyweXWrWsjk4p62jyVcpJfn4y7rt/P3TpInNRx46FMmXkNGpsrNUReo7YWPmb3H23jOUmJED79nDgAPzf/8lpfKV8mWXJ19BBM6+iz9dVISHw1VewfbsUYkVGSgFR8eIwcqRvJ+GEBPjuO/lA0rs3nD8vhVXr18PEidLURCllUfINCAgg1pffobxQbGwsAemWovqWKlVg1SppxlGunCSaHj2gSBFp2nHpktURuk90tHzwuPNO6NRJFqoIDZXF7teskfm7SqmrLEm+ISEhHDt2jJiYGD2i8nCGYRATE8OxY8cICQmxOpysW7iQvz77zGUP53BAixawezfMmydjm5GRMme1SBHo1k2m09jVqVPQt68c0fboIc1JSpSAadNkXLd5c/kbuYyLnz+lrGLJyEuePHkAOH78OAkJCVaEkGVxcXHktGnvu+v3LSAggMKFC1953rxaUBDJJjxvDgc8/rh0xVqwAAYOlGre0aNhzBho3Bjee0+++nl5pUVyMixfLvu2eDEkJcn1990H/frJ38C0fTTp+VPK3Swre8iTJ49Xv5mHh4dTo0YNq8MwhZ33jXHjKLZnj8wVMoHDIcmnZUvYvFkS77RpsGKFXAoVgnbtZHH4ypVNCcE0e/fC9OmSdM+ckescDnjiCXj7bRnbNZ3Jz59S7qI1h8q3zJxJiJvaVNWqBT/+KNW+X38thUiHD0sV8PDhMte1fXs5bV2jhotPz7rIrl3wyy/yAeLff69eX6wYdO0KHTvKOK/buPH5U8pMzp4cKgD8ClwCDgEvmBaRUjZTpAh8+KFMs1mzRqYp5c4tR5L9+kHNmlC0qCzi8PPP0oTCKidPSqJ99VX5cFCpEnz0kSTenDmlF/OSJXDkiIz1ujXxKmUjzh75jgUuA4WB6sACYDvwt0lxKWU7fn6yOEP9+rJg/JIlMH++HFmeOiWLOHz/vdw2JERO4zZsKJ21KlSQJO7Ko+Nz52S61J9/SgevsDA4duza2wQHy1j2iy/Cww9Djhyu275SvsyZ5JsLeBqoDEQDa4F5wMvAe+aFppR95cghSxi2aiU9o//6S6qlly+XRHj6NPz6q1xS5coFpUvLWHG+fDKvuHBhGUcODJTGFRcvwqVL2Vi7FqKirl4uXpTmIEePwqFDsG+fXH+9wED5cNCoETRtKqfDtSGGUq7nzOfoGsB6IDDNdb2BhsDjt7pTYGCgUcvG/eMiIiLIly+f1WGYws77xrZtJCYmku2++6yO5JYMQ1b+iYyUpBkdDfHxkJjozL23pXytnu4t/fwgKEiSep48cio8ONgzx56v8ILnL6vs/Pqz874BrFq1agvg1D+nMy+zBsDPQJE0170GvAhcX3LYOeUCcqS805kgvFRBwK5dfe28b6D75+10/7yXnfcNoDzgsoXeawAx113XC/gtnfv94aoAPJSd98/O+wa6f95O98972XnfIAP750y18x5kbLhcmuuqocVWSimlVKY4k3wvAbOBT5Diq3rAE8CPJsallFJK2Za/k7dbCTwDfAW0AHqkXJeeLZmMy1vYef/svG+g++ftdP+8l533Dey/f0oppZRSSimllFJKKaV8VzkgDphidSAuNgU4AUQiVeGdrA3HpXIA3yG9vKOAP4FHLY3I9d5CpgbEAxMtjsUV7NyD3W7P1fXs/nqz83tlWh6X65YCa/CggFykEvKiAagAnARqWheOS+UC+gOlkKr4lsibQikrg3Kx1sCTwHjs8YY+DZgB5AbqAxeR/1E7sNtzdT27v97s/F6ZltO5zh3Lej8HRAAr3LAtd/sb+SQOYKRcylgXjktdQt4MDgLJwHzgAPZ6wcwG5gDnrA7EBVJ7sPfjxh7sdmCn5+pm7P56s/N7ZaoM5Tqzk28eZH5wL5O3Y6VxSAewf5HTKgutDcc0hYFQtLmKpwoFkpBTeqm2Y58jX19jx9ebnd8rM5zrzE6+nyLjGEdM3o6V3kB6eTZAPp3H3/7mXikAmApMQl44yvPkRk4zp3URF/aZVW5j19ebnd8rM5zrspJ8w7l6+uD6y1pkWZUmwBdZ2IaV0tu/tJJSrisOdHVjjFnh7P75Id3MLiNFL94iI8+fHUQjn77TyoOMGyrv4a2vN2d543tlejKV67KyUuf1KxpdrztSLHA45efcSEetisC9Wdiuu6S3fzeTDe8Zx3Bm/xzIp7nCSGezBFMjcq3MPH/eLG0P9v9SrtMe7N7Fm19vGeVN75XpaYSH5bogZBnC1Msw4BegkBXBmCAEGWBP/UM3Q4omnrAyKBf7CtiI7KMdZQNyAoORo42cZO0DqdWmIxXPqT3Y7VTtbLfn6mbs+nqz+3ulx+e6/thrqlEhYBVS3RYJ7EDWObaLu5BTtHHIKc3Uy4tWBuVi/bnxlHR/SyPKmgJIRfAl5FO4neb52u25up6dX292f6+8nt1ynVJKKaWUUkoppZRSSimllFJKKaWUUkoppZRSSimllFJKKaWUUkoppZRSSimllFJKKbd5h5uv4PSJlUEppZRSdhbMjc3dTwBlrQxKKaWU8hXvAseA8lYHopRSSvmCPkjiDbU6EKWUUsoXfAAcQU81K6WUUm7RD1nDt4zVgSillFK+4APgLPAA1xZd5bQyKKWUUsquHMBFbj7N6GEL41JKKaWUUkoppZRSSimllFJKKaWUUkoppZRSSimllFJKKaWUUkoppZRSSimllFJKKeU2/w8pPmqmuPELrAAAAABJRU5ErkJggg=="
    },
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg0AAABTCAIAAACTYaNtAAAgAElEQVR4nO2de1xN2fv41zl1uqiUkluldKOUSKnI6aIJTeU2TJJLbhEjwzCYMoWQGsyYxCfExIxQynVGF9EQpXRxjUlX3cjpfjpn77V/f6yf/drffep0SqVm9vuPXp2111577b3XXs9a63nW87AIggAMDAwMDAztwP7cFWBgYGBg6NMwcoKBgYGBQRyMnGBgYGBgEAcjJxgYGBgYxMHIiX4MQRDZ2dmhoaGMMQIDA0PPwciJ/kptbe369estLS3/+ecfFov1uavDwMDwr0X6c1eAoStUV1fb2tq+fv366NGjPj4+HeaHENbU1Pz0009lZWXnzp1j5AoDA4PkMHKi/9Hc3PzFF1+8evXq8OHDPj4+knT6P/30U2Vl5e3bt0eOHNkLNfyXwefz8/PzR40apaqqymYzU3CG/xxMo+9nQAgPHz6cl5c3b9689evXSzgz2LJlS1hYmIGBQU9X718GhDAmJsbDw+P06dMGBgYLFy5sbm7+3JViYOhtGDnRz+DxeCEhIQCA1atXS0lJ9dBVMjIy7ty500OFdyMQwvDwcD6fL2H+0tLSmJgYCKHooZ9//lm0nJaWlm+//TYsLCw8PDwpKenixYvBwcGM1YAYIISBgYGtra2fuyISUVtbe/LkScnzp6SkPH78uM1D6MZ7s21ACHEc751rMXKin3HlypX6+noFBQV7e/sunE4QRIdNOT8/f/ny5bq6ul2qYPcDIYyLizt27NiHDx+o6QRBBAQE5OXlycjISFjUiBEjIiMjz549K/oQMAzz9PSkiZC6urqKiopff/0VAGBubm5iYnL37t1PuJW+C0EQL168CAsLi4iIeP/+/acUFRQUxOFwuqtin05tbe2xY8cuX74sEAio6Q0NDbNnz1ZRUZG8KGNj4/nz57948aLNo0FBQb0mJ/75559t27Z99dVXR48eTUpKampq6tnrEQz9Bwihh4cHAGDatGkQws6eu2DBglmzZok/saSkZMSIEVlZWZ9W024Dx3FPT8/58+fb2Njo6+vjOE4eCg0NdXBwaG1t7VSBVVVVw4cPT0tLo6VDCKdNmxYYGEh7Pg8fPvzw4QP6X0tLy83NrUv30afBcXzz5s3q6up79+6VlZU1NTXtbOuiFgUAoL6mz0ttbe3YsWMDAgKUlZVXrVpFVgzDMGdn5+Dg4M5W9c8//xw1alRNTQ0tvVtuHEIoyZPHcdzW1vbEiRO7du0aPnw4ACAmJuZTrtshjJzoT0AI7ezsAABLlizpmpxwd3cXcyKO43PmzNmwYUPf+c7Pnz8PALhw4QIAQFFRkaxYSUmJvLx8QUFBF8o8evSotrY2n8+npRcUFEhJST179qzNs1JSUjgcTmZmZheu2MeJjY1FfU1RURGbzVZVVcUwrGtF9Sk5ASFctWqVhoZGWFgYAIDa+CMjIw0NDTs7yEBlzpo1a8WKFbTv6BNvHEJYXV29c+dOHR2d+vp68Zn5fL6CggKPxyMIorKyMjo6uqcfOCMn+hMQwqlTpwIAli1b1jU54ebmJubE58+fS0lJlZaWfnJNuwccx11cXOTk5Hg8nr+//59//onSIYR+fn5z5szp2ufR3NysqqoaGRlJS4cQzpkzB60+0Q6Vl5erq6vHxcV14XJ9HHKS2tTURBBEbm7u69evu1xan5ITzc3NSkpKLi4uOTk5O3fufPXqFUrHMMzQ0PDYsWNdKzY1NVVGRqakpISa2OUbhxC+f/8+ODjYwMAgICCgurq6w1NwHB86dOjOnTs7e60uw8iJ/gSE0NbW9lPkhKura3sn4jju4+PD5XL7yEdOEIRQKFRQULC0tKTV+cOHD4qKiufOnetyyatXrx4zZozonV68eJHNZtO6gKamJhsbm7i4OBzHu7wg02eBELq6unI4nG55731KTiBl0vfff09Lv3LlCpvNfv/+fdeKxXFcW1v722+/pSV29sYhhMnJyQsXLhwzZsyZM2dEJ7hiWLRokaysbHtz326H0WP3M5CitbMb5SIiIuzt7ZOSku7cuWNvb//LL7+I5iEIIiYmxtbWVnSLAEEQqH/scrU7C7rikydPmpqaLCwsUGMljyYkJDQ2Njo5ObV5LlrkpZYjWnMul/vixYuysjJaup2dHYTwr7/+opbm7+9/5MiROXPmAACWLl3am8+hpyE7AmlpafJ/8ih6kkg60k5EHSKQoG3Aj/TQLbQJqlVmZiYAYOLEidQaEgTx+++/m5iYtKnBJm+ZmkLLw2azbW1tk5KSumxuBCFMSEiwtLRcvXr1F198kZ+fv2TJEllZWclL+O6771pbW318fHrpwfaCLGLoLiCENjY2AADR5dEOT8QptHnus2fPAAB//PEHLT0jI8PS0lJbW/vIkSPkie/evVu6dGnPrVBFRERwuVwTExMAgK6uLpfLXbBgAXn1b775Rl1dXXQNHcOwjRs3jhgxQl9f/+TJkzwez8nJSUVFxdvbu6GhgZqzpKQEAHDmzBlaCRBCIyMjUv2D4/iqVausra23bNni7e3t6Ojo4+PTQ7f8WcjPz+dyuWpqamw2m8vlcrnc5ORkgiAghC9fvly5cqWTk5ONjY29vf3z58/RM0lNTbWzs7Ozs9PV1Y2NjR0zZoy2tnZ4eDj5xABlWI3jeHR0tIaGho2Nze3bt8nrPnjwYO3atV3WgogHQrh69eqpU6dqamoCAMzNzblc7t69e8kqGRgYeHl5iY7937175+7urqSk5ODgcO/evcePHxsbGw8fPnzfvn20zMeOHQMAVFVVkSkSzifq6+tDQ0P19PRMTEzOnTsnFAq7cIM1NTXBwcHIzO/UqVO9MMdl5ER/gpQT3t7e3d444uPjAQA0PW15ebmRkdHjx483bNgAABAIBCg9NDQUAID6lJ7g6dOniYmJaAgfHh6emJj48OFDdAjH8enTp1tbW4sqEletWuXv719QUJCSkgIAmDhx4pkzZ+Li4jgcDk0bgeO4kpLSli1baIVACOfOnTthwgT0wRcWFnL/L5+y2NUH4fF4t27dsra2BgDcuHEjMTGxqqoKQog69/z8fAghhmEnTpxQVVVNT0/Hcbyqqur06dNDhw6VkZEZM2bMo0ePAAADBw5ET4zWXd68edPZ2bmiomLs2LG2trboaUMIZ86cKSsr2wU1soTcvXv31q1bo0aNUlRUvHXrVmJiIqmcaGlpkZKSQnsdqDQ0NNjY2Jw8ebK6ujowMFBJScnc3DwrK8vPzw8A8ObNG2pmtLsoJSWFTJFETly7dk1TU9Pc3PzSpUtdXppLT09XUVHZtm3bjRs3AADq6urkV9lzMHKiPwEhRJ/08uXLu11OHDp0CABAW5rfsGHDzz//jOysZGVlGxsbUTVmzJhB/uwhIITOzs4AgLq6Omo6hmGjR48WVcjfvHlz4cKFKLG6uhoAMG3aNAzD7OzsWCzWnTt3qJlxHNfS0mpTzePt7a2pqYmGurAteuRuPx8Qwi+//BIA0NzcjFKePn0qJyf3ww8/kHlwHJ89e7aenh45LUMiPCwsDMfxH3/8MSoqisxJdpcQwnHjxj158qSxsVFGRmbatGkovbW1dcCAAQ4ODj36MIVCIYfDsbGxoV2loKAADT6oiRDC77777uTJk+jnuXPnAADBwcHNzc0sFktdXZ3WCHNzcwEA58+fJ1MkkRMbNmwYNmzYpUuXujyRKi0tVVdXX79+PY7jSBsPAMjOzu5aaZLD+HfqZxAEASTWT5SUlPzxxx9iMnz//ffk/zweDwAgLy9PzWBiYuLl5VVWVnb37t1ly5ahoy0tLbdv37a2th4wYIBo9dBYvkOmTZsmPgOEMDs7W19fX1FRkZrOYrF4PB6tngAAGRmZ7du3oyeTlZUFAHB2dpaSkjp69KhAIBg3bhytEGVlZSROaKioqJDp/0GHiQRBREZG8vl8U1NTMpHNZhsbG8fHxyckJHh6ehIEgZRY1tbWbDY7MDCQWgLpQwxC6OPjY2Rk9NtvvwkEAm9vb3RWampqc3Mzkt+0q+M43p4jgOrqahkZGQ6HIyMjY2Rk1KGnstzcXKFQSL0LRF1dHQCA1nRZLJahoeHChQvRT7Tpevr06fLy8nfv3tXS0ho4cCA1/6BBg1CVxNeBxuHDhzdu3BgSErJr165NmzYtXLiQw+FI3sYghEgw79q1Cz1JR0fHgoKC0tLSCRMmdKomnYWRE/0JgqKLIwhCkhYmuZxApdF8gaxatYogCLSL56uvvkKt8+7du62trba2tqIVID4u3YpiZGRE/dmhnCgrK3v37t3UqVNpV0EPgcPh0J6Ao6Mj+X9aWhoAAG1ZNzY2brN8WVlZNEiklS8rKysQCFpaWmjy6d8KtVGhv/n5+QAAWs+Itlg/efKEmllLS6u90gAAUlJSvr6+EMILFy7IyMi4ubmhdGQmwOVyRStz7dq1oqIi/kdaWlr4fH5ra2tLSwsaOyMEAkGHciInJwcAYGpq2mb7EfV5s2rVKjLD/fv3Bw0ahDpfZGFIA+kG6uvr27zx9mCxWKNGjYqIiCgsLAwLC9u7d+/GjRuXLFkyYMAASb5lgiAuX748Z84cUgM/bNgwAICOjk6H534ijJzoT7BYLNIYSZKGNXLkSPS1SIKqqioAgObbAABAEMSVK1fk5eVJTyFJSUkAAKQpocFms9GeuE8nOzsbAGBubk67UxaLpaamxufz23sCEMI7d+4oKiqam5uLKZ/H47X5gSHvIHJycl2ver+CxWJRJ6ksFgt1gjRXEBiGAQBQ8yNPUVJSEi0N2QiQNDY2JicnT506FcldCGFKSoqUlJSFhYVoZdzd3bvrvtCcQPQqgwcPBgCI8UBVX1+fkZHx5ZdfivnERBuJ5NMCFoulp6cXERFRU1Nz+PBhS0tLT09PX1/fQYMGiS8kOzu7pKRk8uTJZLaKigoAgL6+voSX7jKMXWw/o1PrTp1CTU0NAPD27VvRKyKTJ7TUAyFMS0uTkpJycHBosxyWZHRYH7QELNrXEwShpqZWUVFBG8HV1taGh4fX19d/+PDhwYMHU6dORWNGCOHy5ctRN0ctpLa2VkNDQ3SwWVtbO2LEiP+U/3Dqk2SxWKhvraqqouZBXZKFhQV6fej5SDKIzs/PFwgEXC4XnVJXV5eXlzd58uQ2p2sSNp4O2w9BEHl5eRwOZ/z48bRDqJ1XVlbS0p89exYdHY3EGIZh5HS5vLwc7cCgZq6trQUAjBgxosPbF4O6unpwcHBmZqaqquoXX3wRGBiIJrjt5X/69CkAYPTo0egnhPDRo0cuLi7kGizxUaP2KbVqk//Qx/Dv4FPkBFqtau8oWsktLi6mpeM4LhAI1NTU0LmVlZVZWVnW1taiGoJuBH3noK1ZC5vNHjt2LG3QirwprF+/Hhm64DhuZGSEntJff/1VXl5OW2dA4sTAwED0SRYXF7eZ/i8GvVlyN4Cfn9/QoUMvXrxIpggEgqSkJGtra3LtSPKtA42NjeDjbBUA8Oeff0IIHRwcevQJEwSRlZVlamoq6iNSUVFx1KhRtHb+7t07CwsLb2/vlpaW69evA8py5eHDh1VVVWm1LSoqAgBQl8K6jIKCgq+vb0ZGxoQJE7y8vDZv3iw6p0egMRMpv4uKinJzc/fu3UvWbfXq1dra2levXpVEfneObteMM/QcEMJJkyaBztvFYhj2v//9z83NzdfXtz2vDBiGqampkWbmJDiOz5gxQ11dPTs7+8WLF5aWlqCtPa7dC4Zhurq6NMd/JL///juLxaK6wUEeTWbPnv3y5UsTExMzMzN7e/vGxsbo6GgrKyuaUSNBEH/++ScA4ObNm7R0Pp/P4XA2bdrUR3YU9zSPHz8OCAjQ1tYGAKxfvz4gIKC+vh5CeP/+fX19/fnz58fExCQlJX311VdcLre4uJggiPz8fH9/f7Q1wcfHJyAggLqxWdTsp6GhYdCgQVwut6Ki4saNG2g4f+vWrR69L2TUtHr1atHPBMfxZcuW2draUivJ4/FkZWUPHTp05coVU1NTNTW1nTt3VldX//DDD3PnzkUeTahs3LhRVlaWtBAjumkjOo7jubm57VlD4Ti+Zs2aUaNG3b17t6GhwdHREU2AyKMaGhoAAGtr625vvb2tn6ipqREKhZ84X+sCtbW1zc3NqHFLwuPHj8ePHy865Hn16tXQoUNpKr7ehCAI8HGZWPJTDhw4YGJicuLEiZCQEHNz8/T0dFHtLovFcnJyysjIIP6vapfNZsfExJw5c2bnzp3IMxJoRwnZjbS0tBQWFnp6erZ51NHRkcVipaenI8NZVPkDBw5s375927ZtR44cMTExCQoKmj9/vqWlZWJiougyemZmpry8vKiKMicnRygUOjs7/0fWnVBzWrJkCfUni8WysrLKycm5dOnS5cuXBw4cuHjxYldXV3KticVieXt7o3+IjoauioqKmZmZ0dHRixcvVlVVVVVVbWxsbFO51Y0gVbyVlZXoJ8xmsx0dHWNiYnAcJ9/ywIEDDx06FBMTo6end/369crKyl9//XXFihVubm6kcREJQRCZmZnTpk3r1A5qSWCz2TTDPNrR8PDwU6dO/frrr1JSUt9//72TkxN5g2w2++zZsykpKVeuXOn+uZokwoTcu4/+dtmKvLq6GnVSXThXEgQCQXZ2tqjw5/F4lpaWqampkhe1ePFiX19fUbF89uzZCRMmdOjQsYeAEKK145UrV0r+Cvh8voyMzI8//kgQBIZhOjo67u7ubY44UlJSFBUVqQ8Q7aeNj4/HMAxCKBAIdHR0jI2Nu7aPtENwHH/79i2Px0tOTgZtbQ4ns82fP3/dunWiu+SoedobVUEIzczMFi9eLJphx44dGhoanfK08++msx87bVjd2NgYGhqal5eHuo6CggIWi7Vhw4Ye2jkBIXz9+rVAINixY4e0tPTbt2/bzNbc3Kyuro7WZ2iVpw7P22s/5eXl0tLSKIoJ9VzQW46txDy9iooKe3v7bn+8Hc8nCIJYvnx5YWEhNZHFYkVHR4taxYlBIBA4Ozvv2LED7RTrdgoLC93d3ceNG5eUlJSamkqOlzEMmz59+oYNG5BHbgk5cuSIqampgYGBn58fVTh7enrGxsYuXLgwISGh58LJiYHo/Moji8UaNGiQUCgEALDZbENDw2fPnrU54rCzsxs9enRUVNS6detQysOHDxcvXjxixAhXV1cWixUXF1dWVnbr1q0eGm5nZWU5ODjY29tbWVlpaGigzVyisNnsH374wd7eft++fdS5Am0a1N5V7t+///Tp099//52Wp6Wl5dixY3v27On2cWL/5RNHpgcPHty5c2deXt5vv/2G43hYWJiGhsaOHTt6QjlBEER4ePiGDRsOHDhw//79r7/+eujQoW3mlJOT8/Pzi4iIoBk1UduDmPYTERGhq6s7b968zlbv9OnTb968EZ9NRkZm27Zt0tLiemYxT+/q1avTp0/v9sfb8dfOYrFWrVq1dOnSzMzMu3fv6ujorF69OiAgQPI1HAAAQRBHjhzR0dGZO3fuJ9S2XSCEnp6eenp6ra2tNTU1iYmJ5KHIyEhlZWXkOVlylJWVz549u2XLFpqrOBaLFRUVlZmZiXZs9j5E5/XYMjIyFRUVe/bsAQAIBIKcnJz2tIhsNvvMmTOHDx8mI8cpKCgAAI4fPw4hPHfu3IYNGy5cuGBnZ9dDciI9Pb2lpcXCwuLixYv79u0TE6jO1NR05cqVAQEBnbXuEAgEGzdu/PHHH2n7OQAAu3btGjduHGlHz/DpKCoqDhkyZPfu3Twe7/vvv09JSbl9+3Z73fcnQhBEYmKiqqqqvLz848ePd+/e3V4rZbFY3377bUVFBbWjkJDCwsLw8PAzZ8501nJakjU68H+9WHaWp0+fHjp0qEcasITzDqRkl5OTE13VkYTS0lJ5efmeU16lp6cDAPbt2+fj42NpaUn6cqmqqlJUVIyPj+9CmRBCW1vbb775RnQSFxgYaGho2ENrL2LAMMzAwAAAEBYW1rUSDh48OHjwYDJAmyg4jickJCxatAgp0zAMO3jwoKmp6ciRI3fs2FFdXd2jM+v379/PmzcP9SwdXojP5zs6OtIccogHQrh161Zra2vRd5eYmKimptZ3Ym/0U2jLL83NzStXrhwyZIi1tfXRo0d7ekEvPT190qRJBgYG169f73DtpaSkZMKECbW1tZKX39raOm7cuKCgINHG2Rccql+4cIHqbLEbkVROXLx4EQAwderULqx8QQj9/f21tbV7yD0kQRA7d+4EACQnJ9NWFffu3Tt8+PAud+ixsbHy8vKikUNqamoGDBjQ+1FryGE+KQg7RXZ29sSJE/Pz88Vnw3H82LFjVB9/yJlMr7k2kvxatbW169evb2lpkbDk0tLSFStWiAYegBAuXbq0wyfD0CE4jpuamlK/dNJXce+0HwzDJOysIYRZWVkhISGSV+zSpUvt5ccwjHbj/yYklRNr1qwBAIg6WZQE5Lht9erVXThXEnAcnzlzJovFog2TMQwbP368l5dXl0sWCoUDBgyguQwjPoZ2mTt3bi8PH9Bi15gxY7rQHJ89e+bq6srj8SCEsbGx4r+Nf5+3OwaGbuG/+WlItMoMIbx9+zaQwCdPm1RUVLx8+dLKykq02Li4uKioKKqHABzHHz9+jJSuHcLn85OTk1NSUnJzcwcOHBgfH5+cnEzutPzw4UNOTo7odQEAra2tcXFx+/fvv3r1Ko7jpaWlP//887Vr12jXlZKSMjMzS0tLo60Yslgsc3PzO3fudDlQSRfAcfzgwYPIALSz6oHq6uqtW7ceO3ZMSUkJw7DTp0+Lz/+f2mXGwCA5/81PQ6Lupqqq6uXLl3JycmiPVWdBjnrI7eYIHMcXLFiQlpYWGxtraWlJdrhRUVHm5ub379+XpGQej3f8+PGAgIC3b9/KyMicPn363LlzaEs9+OjghXZdAMDbt2+nTJly584dPT29TZs2rVy50sPDY+TIkdu3b0cbc8icLBbLwMAARU+kYWBg8P79+5cvX4qpXpteqUWRUNjExMRkZWXt2rVLvOcZUYRCobu7O/J9LyUlJSMjQ26OZWBgYOgQifbZIauAiRMnIp+RYkhKSnr79q2BgQF1Hw3aG4n2fJKcO3dOS0vr4MGDzs7OBQUFEEIpKSmCIK5duyYtLS2hm9xhw4bFxMQ8ePBg8uTJ3t7e+/bto/p+efXqleh1kbef9evXL1u2DABw7dq106dPnz9/HgDw5MkTUYPIIUOGIIt+Wt+KbDaePn2KYq6JcuPGDeTWn4asrKy8vPyAAQPk5eXl5eXl5OSmT5+OjJHEUFxcvH79+lmzZm3fvr2zk4mWlhZnZ2dySxoAoKe9JjAwMPybkEhOII/wU6ZMEe2hIIRk4tmzZ8vKygoKClasWJGXl0faHb5//x4AoKysTJ5FEERUVFRUVFRtbe2dO3fs7OyQvTCGYUlJSZMmTZLcpTOLxSIdi9KqJ3pdAEBjY+ObN2/QBlSCIN68eSMrK+vu7l5eXm5vb79lyxZaIUOGDAEA1NXV0eQEmd5exVxcXEQnCkRbzsCJjqzl0tLS5s2b5+XldeDAgS5s2hg4cOCuXbs6exYDAwMDouORKYQwNTUVADB58mTaoRcvXsyePZvs5goLCzU1NU+cOCEQCKjG6citFc0Hb3Jyso6OTkxMjFAo/Prrr1HvmZaW1tTUNHXqVMmHzARBINfZoo4h0XVpU4SBAwc+f/6cdF15//59W1tbOTk5fX3927dvu7i40ApBAUlQDB/RdKoDelHYIkhJSbWZKKaQ5ubmmTNnurq6Hjhw4L/j7JqBgaHv0HF3XFFRUVhYyGazRfcznzlzxs3NjRwgOzk5bdmy5f3797QhM+pSaU4Q2Ww2hDA+Pp7D4Xz99dcoEWnLO+X7hSCIx48fKysro40FVFA0D1GVOCmE7t27h+O4vb29mEUY5O1SdD0KpYtfiOsW/YS8vPy1a9euXLni7OxMCwnAwMDA0At0vO6EwlhaWFjQnN/V19dHR0c/e/aMTDEzM5OXl1+2bNm1a9eoPS/yENnc3Iw295IQBJGWlmZtbY1KJggiKyuLxWK1F9igTTAMy8vLE416Bj4GJGlubqall5eXKyoqKisr37t3D1DiVb169aqhoYEW8ABtWVBXV6cVgrTloukkWVlZ3377rSS3MGPGjB07drR3lMVi2dvbp6enW1hYLFy4MC4uTvyefoY+C0EQ//zzj56eHqMcYuhndGg5i/S9W7ZsoSa+f/9+xowZnp6epDXx33//PW3atODgYABAUlISNTPy6ZaVlUUrGS3mLFq0CBXS2Ng4cOBAZPuEMly4cGHFihXit8gi5cTmzZtFD6FN2mlpadTEu3fvysnJzZ49G8MwNHFBu7RwHP/iiy9EN/euX78eOUeipSckJAAAHjx4IKZumMSIKYTk5MmTAIBt27b9R1xe/5t49uzZH3/8YWNj4+bm9t80wGfo17Q7MiUIoqWlJS8vD3nqV1NTq6+vr6ure/Xq1YsXLw4cOFBcXJyUlIRGRgUFBZ6enomJidra2ocPH05PT6futLCwsGCz2Xl5ebShupKSkq6ubnFxMdruu3r16vr6ejLuVWtr6+LFi1tbW6WkpI4dO9beECwrKwsAgKIy0Bg3bhyHw8nLy6O6j87NzeXz+VZWVrGxsc+fP5eVlUU99Y8//iglJUVzNE0QxKtXr8zMzESvXlBQIC0tPXbs2PYeIGgrBu+nsGTJksOHD+/fv9/MzIzU6DD0F7S1tUeOHCk6u5UEHMefPHmSlJSkoaHh4uLyGd3aM/w3aVc/8fLly5kzZ27dutXQ0JDL5d64cWPGjBkODg6urq7r1q0rLi62trYmAyaHhoY6OTkZGhrKysqOGzeOFmNywIABJiYmDx8+FL3K+fPnhwwZ4uLiYmpqipawpkyZgg5xOBxfX187O7tXr16JcYyF5hNTp04VPSQrKztx4kTadb/++vQ/qBkAABaiSURBVGs9Pb2rV6+eOHHi+fPn8+fPt7e319LSysnJOXfuHE1/juP4vXv3HBwcaD0+QRB3797t6ZhuNKSlpbdt2wYA8PPz67KnMIbPgpGRkbW1dZejEIaEhJw9e1ZLSys+Pn7UqFHUxV4Ght6gvYkG0q8KhUL8I0KhkPxJXS1BHl2ioqLQ/xYWFtHR0bTSduzYoaurS1tg4fF4lZWVqCihUGhmZjZixAiqn0EI4Zs3b2bOnNmm1y3kMWbSpEntRT0jCGL37t0aGhq0VSPyjoiPnovQfYmenpCQwGKxnj9/TktH0a+OHDnSzsPrKahbCDt1IvnK2rxNDMN6IgbWJ9Je5IOkpKSIiAjJywkPD79+/Xqbd9fU1ESLa9ZzQAg9PDwkWXeifWJ8Pl9JSQmF4kBh/lxdXfvay2L4d9MNcU8hhMbGxshtHI/HGzp0aGNjIy1PRUWFjIwMVW+Rm5srLS1tZGSEWjzaonH69GnaV3T8+HF/f3/RK4aGhnp4eFRWVkpLSx88eLC9z+bDhw9KSkpXrlzp2n3NmDGjzZA+oaGhQ4YM6f1oRaS/2AMHDkh+VktLy4oVKwYPHsxisaytrUNDQ2kZ+oKrSyo4jicmJs6dO3fbtm01NTXUQ/fu3ZsyZQo13mSH8Hg8PT29NjVJyH6sl+WE+MthGBYUFKSjowMAMDMzW7t2bVNT09ChQ7dt20Z8bJOjRo367EqON2/eBAcH+/v779mz57O3nN50UvnfpHviY586dWrJkiX//PPPokWLTp06JZoBQrh27Vo7OzuySaFg5cuWLSMIIi8vT0tLy8PDgzbwr6qqMjMzq6yspJXW0tICAJCXlz927Njw4cPr6uraqxiEcPPmzRMnTuxCU37z5o2UlFRBQQEtHcMwTU1NUeeAvQD8GM+uzcC/7REZGblp06bKyspDhw4pKSm5uLjQzu1TcgJCmJycrKysfP78eRUVlbVr15K1LSoqGjx4cE5OTmfLTEtLU1dXLyoqoqUj4+belBOurq7iX1xGRsaXX35ZVVUVHR2toaGhqanZ2tpK7Qf19PTc3Nx6ocJU0OoCNeX3339Hi72qqqqft+UkJiYOHjy4951y/qfoHjlBEMSjR49CQkJSUlLa+wx4PJ6JiQkZa7C1tXXmzJnz5s377rvvhg0bFhgYKOqbfs+ePRcuXBAtCsOwGTNmWFlZOTo6pqamiv/w6uvrx48fHxMT06nbQeZPO3fupDU+CGFYWBiXy/0soTFhl+KeBgQELFq0CJ1eUVEhOhjvU3ICx3FDQ0MnJ6ft27cDAAIDA9Gd4ji+dOlSb2/vLtQTQuju7j5nzhzaQ+uDciIuLs7Y2Bj939DQQJtOHT9+XFVV9fXr1z1Y0XZq5evrS0sUCoXGxsafXU7s27cPAKCkpNRHGvC/km6TE5JQVlY2efLkhw8fop8Qwrdv32ZnZ/P5/DY/HjFfFI7jBQUFjY2NknSXlZWVU6ZMoRnIiic4OHjdunWiLe/+/ftWVlafMT42csW4fPlyyeXE+fPn2Wx2RkZGexn6lJzIzc0FAGzatKmlpeXevXvkFLO8vFxGRiY3N7drxaamprLZbNrssKGhoZflxJdffin+xSHPksePHxc99Pjx4wkTJuTl5fVYHdvll19+WbduHS0RKSY/u5z48OFDUFDQ9evXP2Md/vX0qpwgCKK6urqwsLCXL0oQRG1tbadi+zx58qTN7/nly5etra3dV6/O0TU5IRAIVFRUdHV1Gxoa2szQp+REREQE0lTR0nfs2KGjo9PlODA4jmtqam7bto363HpNTiDV9Pz5852cnNozmkBgGDZlyhQFBQWaSHv58qW7uzsKH/L06dPeXI5Hc+s25cSECRM+u5xg6AV6JMqxGNTV1UeNGtXLFwUADBo0SF9fX/L8Y8eObdOK0dDQUEzQ5l6gC7aV0tLSS5cuLSws3Lp1KyFBhF4AAI7jOTk5yB6BTCQ+SpQeAkKIYdijR48AAJMmTUK9DzqEYdiVK1dsbGxEHX8RBNHU1JSRkUGuBBYVFRUXF0ORkCFWVlZ37tyhpkv4ND6d48ePOzg4VFVVCQSCadOmHTlypL2cbDZ75cqVTU1Ny5YtI6sqEAgCAwMjIiKUlJQghIGBgd1bPYIgMAxrbGx88OBBaWkp9S0LhcL9+/cnJiYSHy3Q4P91vI/+wXG8pKSktLRU1GKbIAhU8rNnz9CeVvIUHMeRzCssLCTN36n+bKgXJdsD1ecNtRyyZAhhTU1NcnJyRkZGc3Pzq1evqHeEYdjr168fPHhQWVnZm/Fj+je9LpkYug6yAwYAeHt7SziirKurCwwMHD9+vIyMDPKtK5qHNp9oampycXHx8PAYNGhQbGwseelNmzY5OTn10EgWx3Hkj4tKUFAQOtrS0sJms3fv3i161s2bN42MjNavX29mZnb+/Pn9+/dPmDBh0qRJoiqc0NBQaWlpqjEecuPYa+tOJO3laW1tjYiIMDc3R86JIyMjCYIQCoU012peXl7dWGccx8vKyubNmzd79uytW7eOHTt20qRJqNvFcVw05ExISAh5IqpqYWHh0qVL/fz8hg0bpq2tTZ24C4XC0NBQVVVVPz8/ExOTAQMGnD17FkKIdu8i9u/fb2xsLC0tPXfuXKFQSHVL88MPP4wcOZL8GRoaSj3Rx8cHRXgEAAwcOJC0dP/hhx9MTEzWrFnj4eGhpaXFZrNJU5dXr15ZWlpyuVwfHx8Oh+Pk5CQmVjwDCSMn+hMQQhSeT0I5wePxjIyM3NzceDwecqW+Zs0a0ROpcgLH8XXr1v3yyy8CgUBaWtrDwwPlb21tVVBQoHpV6XYwDGtpaZGWlra0tBQIBNQlJrTr/uzZs7RTMjIynJ2dkbroyJEjcnJyX3zxRUNDA/JtTFshjI+PBwA8evSITKmrqzM3N+8jyyYYhrm5uZmYmJSUlBw8eBAAgDa11NfXB/xf0Oi+u3j58qWKisqpU6fQixYKhb6+vgMHDiwuLka1SktLAwD4+vrStuAgOSEvL79u3To0UUDKlc2bN6OiIITBwcEcDqeqqgrlnzt3rrS0dGFhIYSwtLQUCaHRo0cjA3d5eXmhUIhh2LFjx5AYwDCsqalp69atAICwsDC0ZSo+Pl5GRgaZzKCd6sOGDSPlREZGBgCAtG17/fq1tLQ0khPl5eVDhgz57rvv0KHMzEx0lT7SAPoyjJzoT3RKTmAY5uzsrKWlhbpRFJVv6tSpbW5aNDQ0ROlVVVV6enp8Ph/1Dvv370d50AYXmpuvbgeFIPTx8aHd3bVr1wAA165do1Xbzs7uxYsX6GdAQAAA4ObNm2VlZQoKCnPmzKHdaVJSEgCA2sn2HTkBIfT39+dwOEgtUVNTIysrq6Cg0GV9jIQIhULk4pN6odevXwMAyAeI3GWK6icwDEOeeEjRi+M48tyMTkR9NJfLJQu/ePEimhagn8h3HDIxz8vLIze0fvjwgcPhWFhYoHLQHGLFihWoVVy9etXLy4tsIcgIkJQTZ8+eBQD89NNPAoEAHV2yZElLSwuO42ioRPpwgxCOHj1aRUVF1HsbAw3G82h/gvi4Aotennhdxb17927duhUZGamkpAQAmDRpEpvN5vP5YooFACgrK1+5coXD4Zw9e5bFYpEu31EMkjZd+aILdVh5Z2dn0ilLe+Tl5QEAxo8fT7s15BaJ5m+YzWaHhoYaGhoCACCEGRkZsrKyXC53wIABdXV1LBaLpsxAwa9Q9CqE6GI6FRzHafGdCgoKOByOjIwMh8Mh/5GRkQkKCmqzhLKyssWLF4u5BHKkDwBoaGjYt2/f2rVr0T5KVVVVIyOj/Px8Med2C+Xl5Y8ePZowYQL1WWlqagIA0tPTcRwXEwmGfEdUL2ccDoeMIJCQkIBhWHNzM6lQQYbIz549Q60XNTxjY2MAgKmpKVmIsrKyq6vr5cuXy8vLNTU1Y2NjBw8efPXqVQzDpKWlY2NjPTw8qC2EWkkbGxtFRcXNmzfv3r3bwcFh7ty5J0+elJaWFgqFsbGxAICrV6+iAJ0AADk5OR6PV11dPWLEiK48vv8MjJzoT1D7PvFCgiCIxMREFov11VdfoRQpKSkpKSldXV3xJ8rKyhobG2MYdvHixfHjx2tpaQEAIIS3b9+WlZUVjUECPnZq5M/2fK0jN+/iQXJi4sSJorVqMz+5eg4hvHfvnpWVFfK41aYHRtQrtba2kikdGgXMmDGDz+fz+fyWlhZlZWUul9vhLdCwtraWJFtiYiJaliGryuFw9PX1e9rb44sXLwAAMjIy1GEH0hBUVlY2Njai4DFUqDlRmGFqFBZqhauqqgAA2trapCM4AICrq+vQoUNRNvR3+PDhtEuwWKwFCxZcvnz50qVL69evT0xM3LVrl6+vb1JSkqOjY1pa2v/+9z9qfajn6urqpqamLlu27MmTJ5cvX758+fKpU6du3rzJ5/PR9nt7e3sy3heqGBMuvkMYOfHvhCCIW7dujR49mvQtWlpaKhQKnZycJOl6Hj58WFtbu2rVKtThNjc3p6WlTZkypc3+2sjIiConugyE8PHjxxwOhzq0RLQXSgR87Lays7MbGxu5XC66OzTfoo2FkXcsyUPqSklJdSpkliiamppoF5h40Mtis9mkgGSxWK9fv/b09OxpOTFs2DAAQFNTE/VCSL0vLy9PdUyL5l5NTU1+fn6RkZFoNoCCgFF7ajabjR4+AGDMmDGoNKr3aCqozDadabq6usrLy587d05XV9fR0dHDw+Obb765cOGCQCCwt7en6rppj4jH46mrq+fk5BQVFT148OD48eO3b9++cOHCwoULNTU1y8rKBg8ejBZvGSSnt+1iGXoNPp9PnU0nJSUNHDjQ09OzzcwFBQXUn0+fPgUAkJ9TRkYGjuOOjo7d6yldlOzsbFNTU1HLYxTqCo1PSVpaWtasWbNlyxYI4Y0bNwDF2fDVq1dF14KQnKDaz/Qd+Hy+iorKgAED0M/s7OwPHz588803PS0njIyMhg0bRrNnRepoMs4jErfI2EEgEBQUFNDW66jyGKkN0InOzs6ysrJogyeZobS0FPnsISHask5WUFCYNWtWVlbWnj17vLy8lJWVnZ2d4+Pjz5w5M2vWLDGPJS4uDsUI0NPTW7Ro0V9//aWmpoZCHbu5uQEAkKKbrG10dDTyA0QQxMWLF2kfAgOCkRP9DAk7DhQF79WrV2jEh+P4kSNHNm7cSPZE4qGuJGAYdv78eQBAm4tO3cjbt295PJ6lpaXoPerr6ysoKBQWFlIT09PTjx8//ueffwqFQmTLhFbJ+Hz+Tz/9hJSWVN68ecNms01MTKiJyC99L0B89JIkegi9rNra2vLycgAAhPDnn3+eNWuWaCjfbkdaWjoiIqKuru7o0aMoRSgUBgUFKSgo/Prrr0gAaGtrs1issrIygiCePHkyZswYlI7+olkFQdm7QHz0pDB06NCtW7fW1dWtXbuW+Lj1Ye/evUhLgbIBimk+7Zkg3VhJSQmKSePh4cHj8VJTU52dnak50bYbshwIYUJCAvlaW1tbW1pa9PX12Wx2UFCQkpLS/v37kSAEALx58yY8PBzNkgMCAhYsWDB58mTxWqv/KN2tGGfoQSCEyOITbcISn7murm7KlCm2traXLl3avHmzh4dHe8Yz6DOjmv1UV1fr6OiYm5snJCSsWbNGQUGBtvOgJ0DDTOSgngaE0M3NjeaUu6KiQl5efteuXcuWLfvtt9+cnJzWrl0bHR1tbW2dkJBAez4QQhcXl/Hjx1MfAoqo2Av2Tv/884+Xl9eUKVNMTU0vXbrUpsmZt7e3jo7O6dOnw8PDx48f32u+YXAcj4uLGzZs2Jw5c/z8/MzNzcePH5+fn0+1f920aRMAwMvLy97eHnmXOnfunK2tLRIVlpaWP/7446FDh5Ayhs1m29nZ/frrrwRB8Pn8PXv2yMjIuLu7BwQELF++3N/fH8fx9PR0LpeL1gCNjIy4XK6ouwTkUP3bb7+FH+NdysvL07bF/PHHH1wuF+kbuFxuTExMZGTkkCFDhgwZsn379qioKC6Xu2DBAmT7RBBEbm6umZmZmpqav7+/v7//tGnTSGdZISEhAICxY8f2Bfu3vsb/Nzlg6BcQBGFra3v//v1ly5adOnWqw7mFQCC4efPmixcvJk+ePHny5PZWjXAcl5aWphq3EATB5/NTUlJev35tZ2c3depUCwuLpKSkHl132rNnT0BAwLNnz9rUdkRGRu7YsePt27fUuc6tW7fS09PnzJljYmIiFApv3bpVVFTk5uaGhsDU0yGEKioqfn5+u3fvJhPr6upUVFTEW/V8OhBCGxubP/74Y9SoUdeuXXN3d4+NjSVV1tRsaWlpDx48MDIymj59enuq+56AIAjkTevJkyeTJ082NzeXlpamPkAIYVRU1Lt37zw8PLS1tQEARUVF1OndkCFDpKSkKioqwMd1JC0tLWSKRhBEbW1tSkpKRUXFjBkz0ND+/fv3OTk5SMnBYrEghNbW1jTVEUEQMTExkydPRkuFBEFcuXLFwMAA2UchiouLkRUvqq2+vn59fT2bzVZRUYmPj29ubp4wYYKDgwO1YWMYlpOTk5aWZmBgMH36dHKRkyCI+/fvm5iYKCsr98RD7t98JvnE0BUghCgy69KlS7txXzRtPpGZmTlz5szbt2+jn2iclZCQ0F2Xo1FXVxcREfHu3bs5c+YYGBi0Z8ze3Nysrq7+119/de0qiYmJSkpKNB/1vTOfQGrhxYsXEwSB4/ikSZOsrKx600ETA8Mnwugn+hloRE/02CyQIIiQkJCbN28iVXZVVVVISMiSJUtcXV176HJeXl5r166Nj4//+++//fz82puyyMnJ+fn5HT58uAvLxwRBhIWFBQQEtGez26MoKSnt3LlzzZo1AAA2my0vL19XV9f71WBg6DKMnOhnIGPzd+/e9ZyoQP6RRo8efeHChenTp3t7e0dGRvbQygxBEA0NDRoaGnl5eUOHDvXx8WlvMY3FYm3atOnDhw9IZd0prl69Wlxc7Ovr26PrS2IICgpCiqXq6uqHDx+uXbu2pw2ZGBi6EUY/0c+IjIxcvXr1sGHDSkpKqCv1nwJNPyEQCKKioioqKrS1tZ2cnDQ1NXu0UysqKjpz5oy0tLSPj0+He/HevXv35ZdfXr58WfINtAUFBXPnzr1+/TpaWKfSO/oJKmvWrBEIBCdOnPhcEouBoQswcqKfUVFRgbrIpKSk9rYvdZb379/v3bs3LCysXwxyS0tLMzMzRfXA7XH58uUxY8a0qRuvqKg4dOhQSEhI79z4b7/9lpKScuLECSkpqX7xqBkYEIyc6GcQBLFixYqoqCh7e/vk5GRmWNpfuHjx4ps3b7777jsWi7Vv377t27czooKhv8D0Mv0MFot1+PDhCRMmpKamoq0Gn7tGDB2TmZkZHR2N3lpycvLff//NvDiGfgQzn+iX1NXVubi45ObmnjlzZu7cuczItC/T3Nyso6NTU1NDpvj4+KAQCwwM/QJGTvRX+Hx+aGhocHCwh4fH6dOnP3d1GMQhasvLLBgy9COYxtpfkZOT8/f3f/z48ejRoxlh38dhi/C5a8TA0AmY+QQDAwMDgziYcQ0DAwMDgzgYOcHAwMDAIA5GTjAwMDAwiIOREwwMDAwM4mDkBAMDAwODOBg5wcDAwMAgDkZOMDAwMDCIg5ETDAwMDAziYOQEAwMDA4M4GDnBwMDAwCAORk4wMDAwMIiDkRMMDAwMDOL4fyOO2941dT7fAAAAAElFTkSuQmCC"
    }
   },
   "cell_type": "markdown",
   "id": "ece865c3",
   "metadata": {},
   "source": [
    "Huber损失——平滑平均绝对误差，替代MSE\n",
    "\n",
    "![image.png](attachment:image.png)\n",
    "\n",
    "![image-2.png](attachment:image-2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fbecc8ce",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T08:41:46.272369Z",
     "start_time": "2022-02-24T08:41:46.263370Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b5ab8c07",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T08:44:52.694241Z",
     "start_time": "2022-02-24T08:44:52.670711Z"
    }
   },
   "outputs": [],
   "source": [
    "# 提取数据\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "1d4ce8e5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T08:45:03.526420Z",
     "start_time": "2022-02-24T08:45:03.492392Z"
    }
   },
   "outputs": [],
   "source": [
    "input_shape = X_train.shape[1:]\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6abada87",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T08:41:46.082860Z",
     "start_time": "2022-02-24T08:41:46.072648Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_huber(threshold=1.0):\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = threshold * tf.abs(error) - threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    return huber_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "4ac8da59",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T08:45:04.037980Z",
     "start_time": "2022-02-24T08:45:04.030982Z"
    }
   },
   "outputs": [],
   "source": [
    "model.compile(loss=create_huber(2), optimizer='nadam')  # 指定自定义的损失函数 create_huber()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "45252388",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T08:45:12.892464Z",
     "start_time": "2022-02-24T08:45:11.918003Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 846us/step - loss: 0.7910 - val_loss: 0.4718\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 657us/step - loss: 0.2530 - val_loss: 0.3770\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x201da19dee0>"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36fe399f",
   "metadata": {},
   "source": [
    "## 保存加载包含自定义组件的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "4bfabf47",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T09:11:24.276956Z",
     "start_time": "2022-02-24T09:11:24.226241Z"
    }
   },
   "outputs": [],
   "source": [
    "model.save(r\"model\\my_model_with_a_custom_loss_threshold_2.h5\")\n",
    "# 直接保存不会保存损失函数的阈值\n",
    "model = keras.models.load_model(r\"model\\my_model_with_a_custom_loss_threshold_2.h5\",\n",
    "                                custom_objects={\"huber_fn\": create_huber(2.0)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a6b73f32",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T09:15:14.635322Z",
     "start_time": "2022-02-24T09:15:14.625316Z"
    }
   },
   "outputs": [],
   "source": [
    "# 通过创建keras.losses.Loss类的子类，然后实现get_config()方法来保存参数\n",
    "class HuberLoss(keras.losses.Loss):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config() # 继承keras.losses.Loss的get_config()方法来保存参数\n",
    "        return {**base_config, 'threshold': self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2a363e90",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T09:15:14.853784Z",
     "start_time": "2022-02-24T09:15:14.845783Z"
    }
   },
   "outputs": [],
   "source": [
    "model.compile(loss=HuberLoss(2.), optimizer='nadam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "cf8a56f9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T09:16:34.376211Z",
     "start_time": "2022-02-24T09:16:34.360211Z"
    }
   },
   "outputs": [],
   "source": [
    "model.save(r\"model\\my_model_with_a_custom_loss_threshold_2.h5\")\n",
    "# 保存模型时，keras调用损失实例的get_config()方法，并将配置JSON格式保存到HDF5文件中，加载模型时将参数传递给函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8c172f34",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-24T09:16:34.580495Z",
     "start_time": "2022-02-24T09:16:34.553324Z"
    }
   },
   "outputs": [],
   "source": [
    "model = keras.models.load_model(r\"model\\my_model_with_a_custom_loss_threshold_2.h5\",\n",
    "                               custom_objects={'HuberLoss': HuberLoss})\n",
    "# 无需再指定阈值"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "751a8645-2a5d-46a6-9300-ddd03b0e84f7",
   "metadata": {},
   "source": [
    "## 自定义激活函数、初始化、正则化和约束"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cb740b1d-df0f-43e5-8612-93b2d91fd0f0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-27T09:16:41.186937Z",
     "start_time": "2022-02-27T09:16:34.579924Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6e01e04e-0d49-4ce1-8e4e-7dc2c895c4fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(z): # 相当于tf.nn.softplus(z)\n",
    "    return tf.math.log(tf.exp(z) + 1.0)\n",
    "\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):  # 相当于glorot_initializer(shape)\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weight))\n",
    "\n",
    "def my_positive_weights(weights): # 相当于tf.nn.relu(weights)，约束权重大于0\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2aed24d6-4d63-47bb-90df-26e862d92c22",
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = keras.layers.Dense(30, activation=my_softplus,\n",
    "                          kernel_initializer=my_glorot_initializer,\n",
    "                          kernel_regularizer=my_l1_regularizer,\n",
    "                          kernel_constraint=my_positive_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e54ba886-ff7c-4b3e-99de-47823faea685",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存正则化参数的l1正则化的简单类\n",
    "class MyL1Regularizer(keras.regularizers.Regularizer):\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "    def get_config(self):  # 不用调用父类构造函数\n",
    "        return {'factor': self.factor}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd0a062a-f52d-4e5c-ad6d-1a70b93523e2",
   "metadata": {},
   "source": [
    "> 必须为损失、层（包括激活函数）和模型实现call()方法，或者为正则化、初始化和约束实现__call__()方法，对于指标则情况不同。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04d04c27-3fc5-4239-a9cb-77034e5d8f2d",
   "metadata": {},
   "source": [
    "##  自定义指标"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe48f002-8c4f-4f77-87fe-ec244049ed65",
   "metadata": {},
   "source": [
    "损失（如交叉熵）被梯度下降用来训练模型，因此他们必须是可微的（至少在求值的地方），并且梯度在任何地方都不应为0，另外损失如果不容易被人类理解也没有问题。\n",
    "\n",
    "相反，指标（如准确率）用于评估模型，他们必须更容易被解释，并且可以是不可微的或在各处具有0梯度。\n",
    "\n",
    "大多数情况下，定义一个自定义指标函数与定义一个自定义损失函数完全相同，实际上，我们甚至可以将之前创建的Huber损失函数用作指标（但是建议用MSE或MAE）。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0be3a20c-019d-436a-84f6-744e9b35cd1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='mse', optimizer='nadam', metrics=[create_huber(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4d3225a6-a46e-465a-97dc-be328fea63b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# keras.metrics.Precision类可以跟踪真正的数量和假正的数量\n",
    "precision = keras.metrics.Precision()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b0fc24ff-476c-4bef-878c-ade00b9adb9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7a66b229-b34a-4f54-b7de-6d4f0fd4d014",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 可以调用result()方法来获取指标的当前值，可以使用variables属性查看其变量，并且可以使用reset_states()方法重置这些变量\n",
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a5e57f9a-c743-4314-a896-af76ab5123f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([1.], dtype=float32)>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ea6bb4ac-86a4-4ee4-8f25-9f90d9b9de69",
   "metadata": {},
   "outputs": [],
   "source": [
    "precision.reset_states() # 变量重新设为0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "18b7b181-819e-4c34-aee7-9ad619005e2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a049ec59-aeb0-4739-8743-e3c3808e4a38",
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Metric):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs) # 处理接收基本参数 (e.g., dtype)\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\") # 用于跟踪所有Huber损失的总和\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\") # 用于跟踪目前的实例数\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None): # 用于更新变量\n",
    "        metric = self.huber_fn(y_true, y_pred) # 更新度量指标\n",
    "        self.total.assign_add(tf.reduce_sum(metric)) # 更新损失的总和\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32)) # 更新实例数，tf.cast()于转换数据类型\n",
    "    def result(self):\n",
    "        return self.total / self.count # 计算返回最终结果\n",
    "    def get_config(self): # 确保threshold与模型一起被保存\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6365704e-6062-43cf-a196-556bc4f3bf5e",
   "metadata": {},
   "source": [
    "## 自定义层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bbcf9ed2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:50:21.580799Z",
     "start_time": "2022-03-03T06:50:21.570791Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f88f83e",
   "metadata": {},
   "source": [
    "### 单输入、单输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "33462e62-e660-467c-a127-bc1efba80664",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T04:42:26.946923Z",
     "start_time": "2022-03-03T04:42:26.934923Z"
    }
   },
   "outputs": [],
   "source": [
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units # 神经元个数\n",
    "        self.activation = keras.activation.get(activation) # 传入激活函数名称，'relu'等\n",
    "    def build(self, batch_input_shape):\n",
    "        self.kernel = self.add_weight( # 创建层的变量\n",
    "            name='kernel', shape=[batch_input_shape[-1], self.units],\n",
    "            initializer='glorot_normal')\n",
    "        self.bias = self.add_weight( # 创建偏置项\n",
    "            name='bias', shape=[self.units], initializer='zeros')\n",
    "        super().build(batch_input_shape) # 最后调用父类built()，执行创建层命令（必要）\n",
    "    def call(self, X):\n",
    "        return self.activation(X @ self.kernel + self.bias) \n",
    "        # call()方法执行所需的操作，此时计算X与层内核的矩阵乘积并加上偏置项，对结果应用激活函数，获得层的输出\n",
    "    def compute_output_shape(self, batch_input_shape): \n",
    "        # 返回该层输出的形状，最后一个维度背替换为该层神经元数量，有时可以省略\n",
    "        return tf.TensorShape(batch_input_shape.as_list()[:-1] + [self.units])\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, 'units':self.units, # 保存其他参数（**base_config）加上两个自定义需要的参数\n",
    "                'activation':keras.activations.serialize(self.activation)}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12f0532b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T03:58:25.004077Z",
     "start_time": "2022-03-03T03:58:24.984022Z"
    }
   },
   "source": [
    "### 多输入、多输出"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bad0b61e",
   "metadata": {},
   "source": [
    "Concatenate层一般是多输入或输出，因此在call()和compute_output_shape()应该有对应数量的输入、输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "37e6c944",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T04:42:28.208722Z",
     "start_time": "2022-03-03T04:42:28.202723Z"
    }
   },
   "outputs": [],
   "source": [
    "class MyMultiLayer(keras.layers.Layer):\n",
    "    def call(self, X):\n",
    "        X1, X2 = X\n",
    "        return [X1 + X2, X1 * X2, X1 / X2] # 三个输出\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        b1, b2 = batch_input_shape\n",
    "        return [b1, b1, b1] # 三个输出形状"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a15c6881",
   "metadata": {},
   "source": [
    "### 测试和训练期间行为不同的层"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39471326",
   "metadata": {},
   "source": [
    "创建一个训练期间添加高斯噪声（正则化）但在测试期间不执行任何操作的层\n",
    "\n",
    "> keras.layers.GaussianNoise 功能相同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3c0db689",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T04:52:00.571208Z",
     "start_time": "2022-03-03T04:52:00.558109Z"
    }
   },
   "outputs": [],
   "source": [
    "class MyGaussianNoise(keras.layers.Layer):\n",
    "    def __init__(self, stddev, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.stddev = stddev\n",
    "    def call(self, X, training=None): # training=None 不训练时不加入噪音\n",
    "        if training:\n",
    "            noise = tf.random.normal(tf.shape(X), stddev=self.stddev)\n",
    "            return X + noise\n",
    "        else:\n",
    "            return X\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f86d4f95",
   "metadata": {},
   "source": [
    "## 自定义模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d45525de",
   "metadata": {},
   "source": [
    "加利福尼亚房价回归任务"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e5a06e60",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:52:36.424379Z",
     "start_time": "2022-03-03T06:52:35.983265Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "adfe780f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:53:29.951359Z",
     "start_time": "2022-03-03T06:53:29.928349Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((11610, 4), (11610, 4))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def split_data(data):\n",
    "    columns_count = data.shape[-1]\n",
    "    half = columns_count // 2\n",
    "    return data[:, :half], data[:, half:]\n",
    "\n",
    "X_train_scaled_A, X_train_scaled_B = split_data(X_train_scaled)\n",
    "X_valid_scaled_A, X_valid_scaled_B = split_data(X_valid_scaled)\n",
    "X_test_scaled_A, X_test_scaled_B = split_data(X_test_scaled)\n",
    "\n",
    "# Printing the splitted data shapes\n",
    "X_train_scaled_A.shape, X_train_scaled_B.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "37cefdd7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:53:54.265281Z",
     "start_time": "2022-03-03T06:53:54.251282Z"
    }
   },
   "outputs": [],
   "source": [
    "X_new_scaled = X_test_scaled"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b4dd57d",
   "metadata": {},
   "source": [
    "该模型没有什么意义，只是作为示例"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fe16694",
   "metadata": {},
   "source": [
    "![image-2.png](attachment:image-2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2cc7a140",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:50:31.368090Z",
     "start_time": "2022-03-03T06:50:31.355098Z"
    }
   },
   "outputs": [],
   "source": [
    "# 先建立一个残差层\n",
    "class ResidualBlock(keras.layers.Layer):\n",
    "    def __init__(self, n_layers, n_neurons, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(n_neurons, activation='elu',\n",
    "                                         kernel_initializer='he_normal') for _ in range(n_layers)]\n",
    "        # 建立含有n_layers个密集层的隐藏层\n",
    "        def call(self, inputs):\n",
    "            Z = inputs\n",
    "            for layer in self.hidden: # input在隐藏层循环一轮再输出\n",
    "                Z = layer(Z)\n",
    "            return inputs + Z # 返回原始输入值加上其通过隐藏层后的输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "350bd17d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:50:54.045640Z",
     "start_time": "2022-03-03T06:50:54.026849Z"
    }
   },
   "outputs": [],
   "source": [
    "# 建立模型\n",
    "class ResidualRegressor(keras.Model): # 继承Model类\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = keras.layers.Dense(30, activation='elu', kernel_initializer='he_normal')\n",
    "        self.block1 = ResidualBlock(2, 30) # 第一个残差层，包括2个隐藏层，每层30个神经元\n",
    "        self.block2 = ResidualBlock(2, 30) # 第二个残差层\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "    def call(self, inputs):\n",
    "        Z = self.hidden1(inputs)\n",
    "        for _ in range(1 + 3):\n",
    "            Z = self.block1(Z)\n",
    "        Z = self.block2(Z)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "042b34a0",
   "metadata": {},
   "source": [
    "若要实现保存参数，则在ResidualBlock类和ResidualRegressor类中都实现get_config()方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "34e3e717",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:53:32.819256Z",
     "start_time": "2022-03-03T06:53:32.799258Z"
    }
   },
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "model = ResidualRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e925a0cd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:53:34.206582Z",
     "start_time": "2022-03-03T06:53:32.971024Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 517us/step - loss: 1.8269\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 518us/step - loss: 0.8722\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 530us/step - loss: 0.6647\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 519us/step - loss: 0.5604\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 507us/step - loss: 0.4961\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train, epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b69a5c91",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:55:32.951204Z",
     "start_time": "2022-03-03T06:55:32.783175Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 0s 434us/step - loss: 0.4268\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "abe7df7b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:55:34.434284Z",
     "start_time": "2022-03-03T06:55:34.424285Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"residual_regressor\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               multiple                  270       \n",
      "                                                                 \n",
      " residual_block (ResidualBlo  multiple                 0         \n",
      " ck)                                                             \n",
      "                                                                 \n",
      " residual_block_1 (ResidualB  multiple                 0         \n",
      " lock)                                                           \n",
      "                                                                 \n",
      " dense_5 (Dense)             multiple                  31        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 301\n",
      "Trainable params: 301\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a0d94c0",
   "metadata": {},
   "source": [
    "也可以用顺序API构建"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5e450bd4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T06:58:13.061642Z",
     "start_time": "2022-03-03T06:58:11.731813Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 0s 511us/step - loss: 1.9205\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 515us/step - loss: 0.5843\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 515us/step - loss: 0.4984\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 529us/step - loss: 0.4710\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 0s 514us/step - loss: 0.4561\n",
      "162/162 [==============================] - 0s 505us/step - loss: 0.4414\n"
     ]
    }
   ],
   "source": [
    "block1 = ResidualBlock(2, 30)\n",
    "model = keras.models.Sequential([ # 顺序API构建\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "    block1, block1, block1, block1,\n",
    "    ResidualBlock(2, 30),\n",
    "    keras.layers.Dense(1)\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa37711e",
   "metadata": {},
   "source": [
    "## 基于模型内部的损失和指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a9122381",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T07:05:34.643093Z",
     "start_time": "2022-03-03T07:05:34.637093Z"
    }
   },
   "outputs": [],
   "source": [
    "class ReconstructingRegressor(keras.Model):\n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden = [keras.layers.Dense(30, activation=\"selu\",\n",
    "                                          kernel_initializer=\"lecun_normal\")\n",
    "                       for _ in range(5)]\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "        self.reconstruction_mean = keras.metrics.Mean(name=\"reconstruction_error\")\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        n_inputs = batch_input_shape[-1]\n",
    "        self.reconstruct = keras.layers.Dense(n_inputs)\n",
    "        # super().build(batch_input_shape) # 调用报错\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        self.add_loss(0.05 * recon_loss) # 将重建损失添加到模型的损失列表中，0.05是超参数\n",
    "        if training:\n",
    "            result = self.reconstruction_mean(recon_loss)\n",
    "            self.add_metric(result)\n",
    "        return self.out(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c9163cd3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-03T07:02:47.418143Z",
     "start_time": "2022-03-03T07:02:46.366914Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 639us/step - loss: 0.7088 - reconstruction_error: 0.6006\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 625us/step - loss: 0.4236 - reconstruction_error: 0.3393\n"
     ]
    }
   ],
   "source": [
    "model = ReconstructingRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "y_pred = model.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8e435d1",
   "metadata": {},
   "source": [
    "## 自动微分计算梯度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a517a96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + 2 * w1 * w2\n",
    "w1, w2 = 5, 3\n",
    "eps = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "69c5ab1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.000003007075065"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f(w1 + eps, w2) - f(w1, w2)) / eps # 导数定义求梯度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "897e66e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.000000003174137"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(f(w1, w2 + eps) - f(w1, w2)) / eps # 导数定义求梯度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "767fa78b",
   "metadata": {},
   "outputs": [],
   "source": [
    "w1, w2 = tf.Variable(5.), tf.Variable(3.) # 定义连哥哥变量w1，w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fa4bc5a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2]) # 自动微分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f3f6555c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c54f2e7",
   "metadata": {},
   "source": [
    "调用tape的gradient()后，tape会被立即自动删除，因此如果尝试两次调用gradient()，则会出现异常\n",
    "\n",
    "如果要多次调用，使用超参数tf.GradientTpae(persistent=True)，使用后将tape删除释放资源"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d7b33da8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "dz_dw2 = tape.gradient(z, w2)\n",
    "del tape\n",
    "\n",
    "dz_dw1, dz_dw2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5982ca9",
   "metadata": {},
   "source": [
    "tape 默认只跟踪设计变量的操作，因此如果尝试针对变量以外的其他变量（如c1）计算z的梯度，结果为None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "83a18e5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2]) # 自动微分\n",
    "gradients"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "617e90b9",
   "metadata": {},
   "source": [
    "也可以强制观察其他变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "039d4162",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(c1)\n",
    "    tape.watch(c2)\n",
    "    z = f(c1, c2)\n",
    "\n",
    "gradients = tape.gradient(z, [c1, c2]) # 自动微分\n",
    "gradients"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8254bbce",
   "metadata": {},
   "source": [
    "大多数情况下，反向自动微分只需执行一次正向传播和一次反向传播即可获得所有梯度，如果需要获取单独的梯度，则必须使用tape的jacobian()方法，也可以计算二阶偏导"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f3b001b",
   "metadata": {},
   "source": [
    "有时候需要阻止梯度在神经网络的某些部分反向传播，使用tf.stop_gradient()函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b98710e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + tf.stop_gradient(2 * w1 * w2) # 无法对加号后面部分微分\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "273e4db5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=30.0>, None]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients # 第二个输出为None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5620e413",
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(x):\n",
    "    return tf.math.log(1 + tf.exp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ddd47bec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([100.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "735cc7eb",
   "metadata": {},
   "source": [
    "由于浮点精度误差，自动微分无法计算无穷除以无穷（返回NaN），下面两种方法解决（第一种暂不明确含义，见书P353）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fe83ea50",
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.custom_gradient\n",
    "def my_better_softplus(z):\n",
    "    exp = tf.exp(z)\n",
    "    def my_softplus_gradients(grad):\n",
    "        return grad / (1+ 1 / exp)\n",
    "    return tf.math.log(exp + 1), my_softplus_gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "51f1cbfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 或者\n",
    "def my_better_softplus(z):\n",
    "    return tf.where(z > 30., z, tf.math.log(tf.exp(z) + 1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7e3c798a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1000.], dtype=float32)>,\n",
       " [<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = tf.Variable([1000.])\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_better_softplus(x)\n",
    "\n",
    "z, tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9221f8fd",
   "metadata": {},
   "source": [
    "## 自定义训练循环\n",
    "\n",
    "除非真的需要额外的灵活性，否则应该更倾向于使用`fit()`方法，而不是用自己实现的训练循环"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7faf190c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 建立一个简单的模型，无需编译，后面手动处理循环\n",
    "l2_reg = keras.regularizers.l2(0.05)\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation='elu', kernel_initializer='he_normal',\n",
    "        kernel_regularizer=l2_reg),\n",
    "    keras.layers.Dense(1, kernel_regularizer=l2_reg),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "63ba1d95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 随机按批量（32）取数\n",
    "def random_batch(X, y, batch_size=32):\n",
    "    idx = np.random.randint(len(X), size=batch_size)\n",
    "    return X[idx], y[idx] # 随机按批量（32）取数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d6a26cfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 显示状态，含有进度条，也可以用tqdm库实现\n",
    "def print_status_bar(iteration, total, loss, metrics=None):\n",
    "    metrics = ' - '.join(['{}: {:4f}'.format(m.name, m.result()) for m in [loss] + (metrics or [])])\n",
    "    end = '' if iteration < total else '\\n'\n",
    "    print('\\r{}/{} - '.format(iteration, total) + metrics, end=end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "edf47c14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义一些超参数\n",
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size # 取数次数，实际上不一定能全部取到，甚至有重复取到的情况，但是也可以接受\n",
    "optimizer = keras.optimizers.Nadam(learning_rate=0.01)\n",
    "loss_fn = keras.losses.mean_squared_error\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.MeanAbsoluteError()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "d91beaab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 - mean: 0.636177 - mean_absolute_error: 0.510248\n",
      "Epoch 2/5\n",
      "11610/11610 - mean: 0.603780 - mean_absolute_error: 0.500410\n",
      "Epoch 3/5\n",
      "11610/11610 - mean: 0.627343 - mean_absolute_error: 0.507777\n",
      "Epoch 4/5\n",
      "11610/11610 - mean: 0.615942 - mean_absolute_error: 0.504816\n",
      "Epoch 5/5\n",
      "11610/11610 - mean: 0.625158 - mean_absolute_error: 0.511287\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, n_epochs + 1):\n",
    "    print(\"Epoch {}/{}\".format(epoch, n_epochs))\n",
    "    for step in range(1, n_steps + 1): # 取n次数\n",
    "        X_batch, y_batch = random_batch(X_train_scaled, y_train) # 每次从数据中批量取32个\n",
    "        with tf.GradientTape() as tape:\n",
    "            y_pred = model(X_batch, training=True) # 训练该批次数据\n",
    "            main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred)) # 该批次平均损失\n",
    "            loss = tf.add_n([main_loss] + model.losses) # 损失求和，model.losses包含一个主损失和一个正则化损失\n",
    "        gradients = tape.gradient(loss, model.trainable_variables) # 计算可训练变量的梯度\n",
    "        # 可以在执行梯度下降之前对梯度应用其他变换\n",
    "        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "        # ↑ 执行梯度下降更新权重，在此之后还可以对权重增加其他操作，如增加约束\n",
    "        mean_loss(loss) # keras.metrics.Mean()，求均值\n",
    "        for metric in metrics: # 更新平均损失和指标（不懂）\n",
    "            metric(y_batch, y_pred)\n",
    "        print_status_bar(step * batch_size, len(y_train), mean_loss, metrics) # 显示状态\n",
    "    print_status_bar(len(y_train), len(y_train), mean_loss, metrics)\n",
    "    for metric in [mean_loss] + metrics: # 重置损失和指标\n",
    "        metric.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d75e1cf8",
   "metadata": {},
   "source": [
    "实际上除了优化器，其他部分很少自定义"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65d0db7c",
   "metadata": {},
   "source": [
    "## 自定义优化器\n",
    "\n",
    "下面是一个示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "a8d4a1e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMomentumOptimizer(keras.optimizers.Optimizer): # 继承优化器方法\n",
    "    def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n",
    "        \"\"\"调用super().__init__() 并使用 _set_hyper() 来储存超参数\"\"\"\n",
    "        super().__init__(name, **kwargs)\n",
    "        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate)) # handle lr=learning_rate\n",
    "        self._set_hyper(\"decay\", self._initial_decay)\n",
    "        self._set_hyper(\"momentum\", momentum)\n",
    "    \n",
    "    def _create_slots(self, var_list):\n",
    "        \"\"\"对于每一个模型变量，创造与之相关的优化器变量\n",
    "        TensorFlow 称这些变量为 \"slots\"（槽）.\n",
    "        对于动量优化，我们需要每个模型变量一个动量槽.\n",
    "        \"\"\"\n",
    "        for var in var_list:\n",
    "            self.add_slot(var, \"momentum\")\n",
    "\n",
    "    @tf.function\n",
    "    def _resource_apply_dense(self, grad, var):\n",
    "        \"\"\"\n",
    "        更新槽并对一个模型变量执行一个优化步骤\n",
    "        \"\"\"\n",
    "        var_dtype = var.dtype.base_dtype\n",
    "        lr_t = self._decayed_lr(var_dtype) # handle learning rate decay\n",
    "        momentum_var = self.get_slot(var, \"momentum\")\n",
    "        momentum_hyper = self._get_hyper(\"momentum\", var_dtype)\n",
    "        momentum_var.assign(momentum_var * momentum_hyper - (1. - momentum_hyper)* grad)\n",
    "        var.assign_add(momentum_var * lr_t)\n",
    "\n",
    "    def _resource_apply_sparse(self, grad, var):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {\n",
    "            **base_config,\n",
    "            \"learning_rate\": self._serialize_hyperparameter(\"learning_rate\"),\n",
    "            \"decay\": self._serialize_hyperparameter(\"decay\"),\n",
    "            \"momentum\": self._serialize_hyperparameter(\"momentum\"),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "9d825e1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([keras.layers.Dense(1, input_shape=[8])])\n",
    "model.compile(loss=\"mse\", optimizer=MyMomentumOptimizer())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6561d2da",
   "metadata": {},
   "source": [
    "# Tensorflow函数和图 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "7ca2fce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cube(x):\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "2bcb68da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "df6e753d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(tf.constant(2.))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1b77a11",
   "metadata": {},
   "source": [
    "转化为tf函数，即生成等效的计算图，TF可以优化计算图，修建未使用的节点，在keras中使用时，会自动将函数转化为TF函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "0cfb4c40",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_cube = tf.function(cube)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "d54a6d5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=8>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "9d07e8f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 可以使用tf.function作为修饰器，效果相同\n",
    "@tf.function\n",
    "def tf_cube(x):\n",
    "    return x ** 3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc7caa2e",
   "metadata": {},
   "source": [
    "在本示例中几乎没有需要优化的东西，因此`tf_cube()`的运行实际上比`cube()`慢得多"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5101589",
   "metadata": {},
   "source": [
    "## 自动图和跟踪"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ec28c31",
   "metadata": {},
   "source": [
    "书P358"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3c26089",
   "metadata": {},
   "source": [
    "## TF函数规则"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "856ad3bf",
   "metadata": {},
   "source": [
    "书P359"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "358.002px",
    "width": "343.021px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "212.197px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "toc-autonumbering": true,
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
